cloud-mu 3.2.0 → 3.3.0

data/modules/mu/config/server.rb

@@ -546,7 +546,7 @@ module MU
           "additionalProperties" => false,
           "description" => "Create individual server instances.",
           "properties" => {
-              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+              "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true, embedded_type: "server"),
               "bastion" => {
                 "type" => "boolean",
                 "default" => false,

data/modules/mu/config/vpc.rb

@@ -417,6 +417,7 @@ module MU
         using_default_cidr = false
         if !vpc['ip_block']
           if configurator.updating and configurator.existing_deploy and
+             configurator.existing_deploy.original_config and
              configurator.existing_deploy.original_config['vpcs']
             configurator.existing_deploy.original_config['vpcs'].each { |v|
               if v['name'].to_s == vpc['name'].to_s

data/modules/mu/defaults/AWS.yaml

@@ -1,5 +1,5 @@
 ---
-rhel71: &4
+rhel71: &5
   us-east-1: ami-0f05fce24aa75ba9f
   ap-northeast-1: ami-0c0ec19eb19055763
   ap-northeast-2: ami-0717ac5c67c99f745
@@ -16,23 +16,23 @@ rhel71: &4
   us-east-2: ami-02f6682c7816b3cfc
   us-west-1: ami-04898e596c06e802b
   us-west-2: ami-02db5457189a8a8c2
-centos6: &3
-  us-east-1: ami-0ccdc671f12147a1d
-  us-east-2: ami-00d0e8bc2f05ab949
-  ap-northeast-1: ami-0726801ceef87f5f8
-  ap-northeast-2: ami-05fa4afc4a0493b0a
-  ap-south-1: ami-0d6e4f3b6592b3139
-  ap-southeast-1: ami-0c988e3dc80b14653
-  ap-southeast-2: ami-02ac856fd094675ef
-  ca-central-1: ami-0ce7e343953af2292
-  eu-central-1: ami-0ce8317423cea27b8
-  eu-north-1: ami-0a923b493d5fc9743
-  eu-west-1: ami-06e0f02328921c865
-  eu-west-2: ami-07ae118c8814df140
-  eu-west-3: ami-03c1017cd1ccc6e9d
-  sa-east-1: ami-05212ae133b9c3ba1
-  us-west-1: ami-0b05ec54412b9f8b0
-  us-west-2: ami-0447e036b102b2ca0
+centos6: &4
+  us-east-1: ami-009723c5c7f8fbc75
+  us-east-2: ami-0781f11395714cd39
+  ap-northeast-1: ami-07fa5a8795da2b6bc
+  ap-northeast-2: ami-0219f0a7c979ff63f
+  ap-south-1: ami-0f24817242c401740
+  ap-southeast-1: ami-042ef2e0643e8e207
+  ap-southeast-2: ami-09fc51de648afa168
+  ca-central-1: ami-0dc643db74edc5aa5
+  eu-central-1: ami-0628759cb297569d5
+  eu-north-1: ami-0aed023791f886315
+  eu-west-1: ami-0f87f0f252ff03622
+  eu-west-2: ami-00abb555d5a460afe
+  eu-west-3: ami-0ccd93d454c2418a2
+  sa-east-1: ami-01e10ea6ea72534ae
+  us-west-1: ami-01fee56b9ee690ffe
+  us-west-2: ami-08bcdb944f185e2a8
 centos7:
   us-east-1: ami-067256ca1497c924d
   ap-northeast-1: ami-07c1e51354fdfd362
@@ -50,7 +50,7 @@ centos7:
   us-east-2: ami-0292786917d1e3015
   us-west-1: ami-0ba622529dcdff2bb
   us-west-2: ami-079a309ca6261d7f6
-ubuntu16: &2
+ubuntu16: &3
   us-east-1: ami-bcdc16c6
   us-west-1: ami-1b17257b
   us-west-2: ami-19e92861
@@ -89,7 +89,7 @@ win2k12r2: &1
   ap-northeast-2: ami-0368c224de1d20502
   ap-southeast-1: ami-028ef74e1edc3943a
   ap-southeast-2: ami-09e03eab1b1bc151b
-win2k16: &5
+win2k16: &2
   us-east-1: ami-02801a2c8dcbfb883
   us-east-2: ami-0ca4f779a2a58a7ea
   ca-central-1: ami-05d3854d9d6e9bcc5
@@ -137,9 +137,9 @@ amazon:
   ap-southeast-1: ami-b953f2da
   ap-southeast-2: ami-db704cb8
 win2k12: *1
-windows: *5
-ubuntu: *2
-centos: *3
-rhel7: *4
-rhel: *4
-linux: *3
+windows: *2
+ubuntu: *3
+centos: *4
+rhel7: *5
+rhel: *5
+linux: *4

data/modules/mu/deploy.rb

@@ -312,6 +312,17 @@ module MU
 
         @mommacat.save!
 
+        # XXX Functions have a special behavior where we re-invoke their groom
+        # methods one more time at the end, so we can guarantee their
+        # environments are fully populated with all sibling resource idents
+        # regardless of dependency order. This is, obviously, a disgusting
+        # hack, and we should revisit our dependency language in the next big
+        # release.
+        if !@main_config["functions"].nil? and
+            @main_config["functions"].size > 0
+          createResources(@main_config["functions"], "groom")
+        end
+
       rescue StandardError => e
         MU.log e.class.name, MU::ERR, details: caller
 
@@ -733,7 +744,9 @@ MESSAGE_END
           sleep 10+Random.rand(20)
           retry
         end
+
       end
+
     end
 
   end #class

data/modules/mu/master.rb

@@ -880,5 +880,26 @@ module MU
       end
     end
 
+    # Recursively zip a directory
+    # @param srcdir [String]
+    # @param outfile [String]
+    def self.zipDir(srcdir, outfile)
+      require 'zip'
+      ::Zip::File.open(outfile, ::Zip::File::CREATE) { |zipfile|
+        addpath = Proc.new { |zip_path, parent_path|
+          Dir.entries(parent_path).reject{ |d| [".", ".."].include?(d) }.each { |entry|
+            src = File.join(parent_path, entry)
+            dst = File.join(zip_path, entry).sub(/^\//, '')
+            if File.directory?(src)
+              addpath.call(dst, src)
+            else
+              zipfile.add(dst, src)
+            end
+          }
+        }
+        addpath.call("", srcdir)
+      }
+    end
+
   end
 end

data/modules/mu/mommacat.rb

@@ -811,6 +811,7 @@ MAIL_HEAD_END
 
       threads = []
       update_servers.each { |sibling|
+        next if sibling.config.has_key?("groom") and !sibling.config["groom"]
         threads << Thread.new {
           Thread.abort_on_exception = true
           Thread.current.thread_variable_set("name", "sync-"+sibling.mu_name.downcase)

data/modules/mu/mommacat/daemon.rb

@@ -288,8 +288,8 @@ module MU
 
     # Path to the PID file used by the Momma Cat daemon
     # @return [String]
-    def self.daemonPidFile
-      base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
+    def self.daemonPidFile(root = false)
+      base = ((Process.uid == 0 or root) and !MU.localOnly) ? "/var" : MU.dataDir
       "#{base}/run/mommacat.pid"
     end
 
@@ -306,8 +306,14 @@ module MU
           Dir.mkdir(dir)
         end
       }
-      return 0 if status
+      if (Process.uid != 0 and
+           (!$MU_CFG['overridden_keys'] or !$MU_CFG['overridden_keys'].include?("mommacat_port")) and
+            status(true)
+         ) or status
+        return 0
+      end
     
+      File.unlink(daemonPidFile) if File.exists?(daemonPidFile)
       MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}"
       origdir = Dir.getwd
       Dir.chdir(MU.myRoot+"/modules")
@@ -346,12 +352,12 @@ module MU
 
     # Return true if the Momma Cat daemon appears to be running
     # @return [Boolean]
-    def self.status
+    def self.status(root = false)
       if MU.inGem? and MU.muCfg['disable_mommacat']
         return true
       end
-      if File.exist?(daemonPidFile)
-        pid = File.read(daemonPidFile).chomp.to_i
+      if File.exist?(daemonPidFile(root))
+        pid = File.read(daemonPidFile(root)).chomp.to_i
         begin
           Process.getpgid(pid)
           MU.log "Momma Cat running with pid #{pid.to_s}", (@@notified_on_pid[pid] ? MU::DEBUG : MU::INFO) # shush
@@ -360,7 +366,7 @@ module MU
         rescue Errno::ESRCH
         end
       end
-      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile
+      MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile(root)
       false
     end
     

data/modules/mu/providers/aws.rb

@@ -844,6 +844,8 @@ end
         @@instance_types
       end
 
+      @@certificates = {}
+
       # AWS can stash API-available certificates in Amazon Certificate Manager
       # or in IAM. Rather than make people crazy trying to get the syntax
       # correct in our Baskets of Kittens, let's have a helper that tries to do
@@ -852,21 +854,24 @@ end
       # @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised.
       # @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
       # @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
-      def self.findSSLCertificate(name: nil, id: nil, region: myRegion)
-        if name.nil? and name.empty? and id.nil? and id.empty?
+      def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
+        if (name.nil? or name.empty?) and (id.nil? or id.empty?)
           raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
         end
+        if id and @@certificates[id]
+          return [id, @@certificates[id]]
+        end
 
         if !name.nil? and !name.empty?
           matches = []
-          acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates(
+          acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates(
             certificate_statuses: ["ISSUED"]
           )
           acmcerts.certificate_summary_list.each { |cert|
             matches << cert.certificate_arn if cert.domain_name == name
           }
           begin
-            iamcert = MU::Cloud::AWS.iam.get_server_certificate(
+            iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate(
               server_certificate_name: name
             )
           rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity
@@ -876,32 +881,45 @@ end
             matches << iamcert.server_certificate.server_certificate_metadata.arn
           end
           if matches.size == 1
-            return matches.first
+            id = matches.first
           elsif matches.size == 0
-            raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            if raise_on_missing
+              raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
+            else
+              return nil
+            end
           elsif matches.size > 1
             raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."            
           end
         end
 
+        domains = []
+
         if id.match(/^arn:aws(?:-us-gov)?:acm/)
-          resp = MU::Cloud::AWS.acm(region: region).get_certificate(
+          resp = MU::Cloud::AWS.acm(region: region).describe_certificate(
             certificate_arn: id
           )
-          if resp.nil?
+
+          if resp.nil? or resp.certificate.nil?
             raise MuError, "No such ACM certificate '#{id}'"
           end
+          domains << resp.certificate.domain_name
+          if resp.certificate.subject_alternative_names
+            domains.concat(resp.certificate.subject_alternative_names)
+          end
         elsif id.match(/^arn:aws(?:-us-gov)?:iam/)
           resp = MU::Cloud::AWS.iam.list_server_certificates
           if resp.nil?
             raise MuError, "No such IAM certificate '#{id}'"
           end
           resp.server_certificate_metadata_list.each { |cert|
+
             if cert.arn == id
               if cert.expiration < Time.now
                 MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN
               end
-              return id
+              @@certificates[id] = [cert.server_certificate_name]
+              return [id, [cert.server_certificate_name]]
             end
           }
           raise MuError, "No such IAM certificate '#{id}'"
@@ -909,7 +927,56 @@ end
           raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM"
         end
 
-        id
+        @@certificates[id] = domains.uniq
+        [id, domains.uniq]
+      end
+
+      # Given a domain name and an ACM or IAM certificate identifier, sort out
+      # whether the domain name is "covered" by the certificate
+      # @param name [String]
+      # @param cert_id [String]
+      # @return [Boolean]
+      def self.nameMatchesCertificate(name, cert_id)
+        _id, domains = findSSLCertificate(id: cert_id)
+        return false if !domains
+        domains.each { |dom|
+          if dom == name or
+             (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/)
+            return true
+          end
+        }
+        false
+      end
+
+      # Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
+      # look up and validate the specified certificate. This is intended to be
+      # invoked from resource implementations' +validateConfig+ methods.
+      # @param certblock [Hash,MU::Config::Ref]: 
+      # @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
+      # @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
+      # @return [Boolean]
+      def self.resolveSSLCertificate(certblock, region: nil, credentials: nil)
+        return false if !certblock
+        ok = true
+
+        certblock['region'] ||= region if !certblock['id']
+        certblock['credentials'] ||= credentials
+        cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
+          name: certblock["name"],
+          id: certblock["id"],
+          region: certblock['region'],
+          credentials: certblock['credentials']
+        )
+
+        if cert_arn
+          certblock['id'] ||= cert_arn
+        end
+
+        ['region', 'credentials'].each { |field|
+          certblock.delete(field) if certblock[field].nil?
+        }
+
+        [cert_arn, cert_domains]
       end
 
       # Amazon Certificate Manager API
@@ -1029,6 +1096,14 @@ end
         @@cloudwatchlogs_api[credentials][region]
       end
 
+      # Amazon's CloudWatchEvents API
+      def self.cloudwatchevents(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudwatchevents_api[credentials] ||= {}
+        @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
+        @@cloudwatchevents_api[credentials][region]
+      end
+
       # Amazon's CloudFront API
       def self.cloudfront(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1117,6 +1192,14 @@ end
         @@dynamo_api[credentials][region]
       end
 
+      # Amazon's DynamoStream API
+      def self.dynamostream(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@dynamostream_api[credentials] ||= {}
+        @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials)
+        @@dynamostream_api[credentials][region]
+      end
+
       # Amazon's Pricing API
       def self.pricing(region: MU.curRegion, credentials: nil)
         region ||= myRegion
@@ -1165,6 +1248,14 @@ end
         @@kms_api[credentials][region]
       end
 
+      # Amazon's CloudFront API
+      def self.cloudfront(region: MU.curRegion, credentials: nil)
+        region ||= myRegion
+        @@cloudfront_api[credentials] ||= {}
+        @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials)
+        @@cloudfront_api[credentials][region]
+      end
+
       # Amazon's Organizations API
       def self.orgs(credentials: nil)
         @@organizations_api ||= {}
@@ -1461,6 +1552,7 @@ end
           require "aws-sdk-core/ecs"
           require "aws-sdk-core/eks"
           require "aws-sdk-core/cloudwatchlogs"
+          require "aws-sdk-core/cloudwatchevents"
           require "aws-sdk-core/elasticloadbalancing"
           require "aws-sdk-core/elasticloadbalancingv2"
           require "aws-sdk-core/autoscaling"
@@ -1481,13 +1573,17 @@ end
 
             if !retval.nil?
               begin
-              page_markers = [:marker, :next_token]
+              page_markers = {
+                :marker => :marker,
+                :next_token => :next_token,
+                :next_marker => :marker
+              }
               paginator = nil
               new_page = nil
-              [:next_token, :marker].each { |m|
+              page_markers.each_key { |m|
                 if !retval.nil? and retval.respond_to?(m)
                   paginator = m
-                  new_page = retval.send(paginator)
+                  new_page = retval.send(m)
                   break
                 end
               }
@@ -1506,12 +1602,12 @@ end
                     if new_args.is_a?(Array)
                       new_args << {} if new_args.empty?
                       if new_args.size == 1 and new_args.first.is_a?(Hash)
-                        new_args[0][paginator] = new_page
+                        new_args[0][page_markers[paginator]] = new_page
                       else
                         MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
                       end
                     elsif new_args.is_a?(Hash)
-                      new_args[paginator] = new_page
+                      new_args[page_markers[paginator]] = new_page
                     end
 
                     MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args
@@ -1535,7 +1631,7 @@ end
             end
 
             return retval
-          rescue Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
+          rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
             if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
               MU.log e.inspect, MU::ERR
               raise e
@@ -1577,6 +1673,7 @@ end
       @@wafglobal = {}
       @@waf = {}
       @@cloudwatchlogs_api = {}
+      @@cloudwatchevents_api = {}
       @@cloudfront_api = {}
       @@elasticache_api = {}
       @@sns_api = {}
@@ -1595,6 +1692,8 @@ end
       @@kms_api ={}
       @@organization_api ={}
       @@dynamo_api ={}
+      @@dynamostream_api ={}
+      @@cloudfront_api ={}
     end
   end
 end