cloud-mu 2.0.0.pre.beta2 → 2.0.0.pre.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 565d062e93b6512b12c242ebc5bdf3c53f3d12d269ae2586b3d77d412eff8844
-  data.tar.gz: c2ffd33d778e0962fe94ec2fdbffa36a5db7d1d9f81754e4f4fa2fb301aace14
+  metadata.gz: cec0ccf09208e41e9134e3c91298505d6e0166f5f08ff6bc36be3b470bbb99e9
+  data.tar.gz: 67af0a4e21700506873ae591641acd7c4272c524cfe61b62f87e5bae98ec2423
 SHA512:
-  metadata.gz: 45d7bb74d8f5de582fc51264b3f58b2e01dd45ee55829a187d3a66e3ea700552b17fe73f9259bb834e1af72379979586cc31066d837826281a1daf073f52006e
-  data.tar.gz: 2409873627fed8e0710264cf73b5a821120558d7f8d3e536e140a5939866fdd8393bdb7b41ce7f3ba0da709cacf288d1b1f1ba1ebaae8b6967842262cd03c13d
+  metadata.gz: ef03c420882a307a26ef726dde97c40f571e21d7ec03727f3b28c10480da770c898a2edee1c30301f1178596a19c6bc0cc724d6a8ad43c44d75835321387f28e
+  data.tar.gz: 0db7d823010c03a83da7040e50286321aed63967c8519e9e5b4791086fb0cf6c0ef63d56d45edb4e57c67fa49288c16feda14281611e42b5c3013fbbc905829e

data/Berksfile.lock

@@ -88,7 +88,7 @@ GRAPH
     mu-master (>= 0.0.0)
     mu-tools (>= 0.0.0)
     mu-utility (>= 0.0.0)
-  mu-master (0.9.2)
+  mu-master (0.9.3)
     apache2 (< 4.0)
     bind (~> 2.2.0)
     bind9-ng (~> 0.1.0)

data/cloud-mu.gemspec

@@ -17,7 +17,7 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '2.0.0-beta2'
+  s.version     = '2.0.0-beta3'
   s.date        = '2019-03-01'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
@@ -27,7 +27,8 @@ The eGTLabs Mu toolkit for unified cloud deployments. This gem contains a minima
 
 It will attempt to autodetect when it's being run in a virtual machine on a known cloud provider and activate the appropriate API with machine-based credentials. Installing this gem on an Amazon Web Service instance, for example, should automatically enable the MU::Cloud::AWS layer and attempt to use the machine's IAM Profile to communicate with the AWS API.
 
-require 'mu'
+
+require 'cloud-mu'
 
 
 EOF
@@ -57,7 +58,7 @@ EOF
   s.add_runtime_dependency 'net-ssh', "~> 4.2"
   s.add_runtime_dependency 'net-ssh-multi', '~> 1.2', '>= 1.2.1'
   s.add_runtime_dependency 'googleauth', "~> 0.6"
-  s.add_runtime_dependency 'google-api-client', "~> 0.25"
+  s.add_runtime_dependency 'google-api-client', "~> 0.28.4"
   s.add_runtime_dependency 'rubocop', '~> 0.58'
   s.add_runtime_dependency 'addressable', '~> 2.5'
   s.add_runtime_dependency 'slack-notifier', "~> 2.3"

data/cookbooks/mu-master/templates/default/mu.rc.erb

@@ -2,8 +2,8 @@
 # user. Regular users get a .murc installed by mu-user-manage, from the template
 # in <%= @installdir %>/lib/install/user-dot-murc.erb
 export PATH="<%= @installdir %>/bin:/usr/local/ruby-current/bin:${PATH}:/opt/opscode/embedded/bin"
-#export MU_INSTALLDIR="<%= @installdir %>"
-#export MU_DATADIR="<%= @installdir %>/var"
+export MU_INSTALLDIR="<%= @installdir %>"
+export MU_DATADIR="<%= @installdir %>/var"
 
 alias vi=vim
 export EDITOR=vim

data/cookbooks/mu-tools/files/default/Mu_CA.pem

@@ -1,8 +1,8 @@
 -----BEGIN CERTIFICATE-----
-MIIF2zCCA8OgAwIBAgIJAOx2Krw2HGFkMA0GCSqGSIb3DQEBDQUAMF0xFjAUBgNV
+MIIFvzCCA6egAwIBAgIJAKAgqiKox1coMA0GCSqGSIb3DQEBDQUAMF0xFjAUBgNV
 BAMMDTU0LjE3NS44Ni4xOTQxIDAeBgNVBAsMF011IFNlcnZlciA1NC4xNzUuODYu
-MTk0MRQwEgYDVQQKDAtlR2xvYmFsVGVjaDELMAkGA1UEBhMCVVMwHhcNMTkwMjI3
-MTQ0ODA2WhcNMjExMjE3MTQ0ODA2WjBdMRYwFAYDVQQDDA01NC4xNzUuODYuMTk0
+MTk0MRQwEgYDVQQKDAtlR2xvYmFsVGVjaDELMAkGA1UEBhMCVVMwHhcNMTkwMzEx
+MTk1MzA4WhcNMjExMjI5MTk1MzA4WjBdMRYwFAYDVQQDDA01NC4xNzUuODYuMTk0
 MSAwHgYDVQQLDBdNdSBTZXJ2ZXIgNTQuMTc1Ljg2LjE5NDEUMBIGA1UECgwLZUds
 b2JhbFRlY2gxCzAJBgNVBAYTAlVTMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC
 CgKCAgEAo7rntOFj/WPNvh00SN55aJBusppsY9arq7QF5gt/9+cBPsjcXn7jJMu0
@@ -15,20 +15,19 @@ PO/6Svt8zTH3pEJMbxwtiwJ8cCLiqSoxj8hOKvvsSmvboN9DwN73JQjOY/pXHaU1
 HLF0qlnfZhU4uiE8+wU1h6oeGZG9fLV63wlGdUXA+HermzovuJ0d2ocy0O93QQDt
 Y92dr6UcPfAmzFyX3Rj9FFMYb2/n1G8l5pEd/Qkx3sH04aoxEmyQU0zugo3zQsL9
 KNyIbp2BTlSh2R/4hWJpWiXFliRvotiJu1s2wdNQ1D3SZgxDbfxf/3j04xgdi5eW
-e4Q3VnxhRfmkS1NqEzIvPabVLg9qvN419cubpE6HAtBJw/f3ocUCAwEAAaOBnTCB
-mjBKBgNVHREEQzBBhwQ2r1bCgglsb2NhbGhvc3SHBH8AAAGCGXN0YW5nZS1tdS1k
-ZXYucGxhdGZvcm0tbXWCDXN0YW5nZS1tdS1kZXYwHQYDVR0OBBYEFK/EmtGebCwd
-5QpM8y/3EKdYNVbcMB8GA1UdIwQYMBaAFK/EmtGebCwd5QpM8y/3EKdYNVbcMAwG
-A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIBACrhE0wT4DBanIUiWNU74e5k
-tH4DC2vOfdPuF7AOFpDDmvR7HHoztWMyV7+KjrsQy/khAWNDq+JJ9Ya1E7mo9Jhy
-w3Ty9xM1FfA39D1dXoGFhfG++HbQSqVXvVWmwa4M/9VmL5YgqCkeEiRf8WQ7TpjT
-ldQUovcIICRAkeZG/lqjHJ/jAADIepCbsOeP0Gs2T0jdLmN2jKkcwFZkqpEi8PQB
-wyhtJvUyU/B7xVBI5etrqmwxApg5RmkmnnsBXAxtSnJP1SvLv0o1IQZfOKzR9Ivn
-IAJaXAp9q2AecHMQhu6TMpNCgifLKcwsGBlmrL+6wgU6+AedWYl80iewzk2kYrEw
-MpIRETKBEOOsIo0jaNzdK/SyxdBtol7T9s6cwGXSr2n06Vtqvwc5d94gqgZyGBgg
-GYnBVfVvRTxGP+2rHry1DRPLlr4/pwzo1XkCPb4QViaYL3jtuQUrj3a6EZZJnS4V
-c9U7RFY8t9uGuIhHN8ZzeZyF0TaAwsMJtkvriRt2rK5BoBn5n6273QOey1YmqnfW
-Xh919XQbuEha1XMG0xRD2RYURJjfSFV81humvQ2vSfZZqZplieXI98TOStRN6cmn
-RE0Zx1kpQ5VVsukYCvQYZ1FzR2PmtOum4IrWQJRWdK8k7hrW+b9YzFDlcimf4Yiv
-uaxTeq345Wu1zn+Pj9AI
+e4Q3VnxhRfmkS1NqEzIvPabVLg9qvN419cubpE6HAtBJw/f3ocUCAwEAAaOBgTB/
+MC8GA1UdEQQoMCaHBDavVsKCCWxvY2FsaG9zdIcEfwAAAYINc3RhbmdlLW11LWRl
+djAdBgNVHQ4EFgQUr8Sa0Z5sLB3lCkzzL/cQp1g1VtwwHwYDVR0jBBgwFoAUr8Sa
+0Z5sLB3lCkzzL/cQp1g1VtwwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQ0FAAOC
+AgEAYtAARdgURxiZ+GaX9R9vpxEjF0NS8cF4k6Z02FKLQV0uO8r1xuv/KA3K2uWd
+huy5IYsN/0q9EtcMWyrKhAOLV92V2y6pojZDLN3R8CYkgoi7HVBALkp3LIwCVyK6
+vY3V0n3op67vWtPNV/lpvSYDb0thUS/SNWk1o7VQxgBYEhV8lTgWKwUN3ViZIS7a
+X4hY47q8pd4f8YztNE7ov56sQBcOFK9ZY0x/96eXWrsLlNdQ85g/6Lzc3jIKrKpd
+dpWMybnEpWSAS6gi6OcUX4JT6K32nQdKvFRZmYl4227BvMODr+iZCBordzUaMp/G
+9nHM8xgUVWwb8asDExtpOELTznZmBMnn5yosv6cklASGWcX0DwJPo2f/BlPKg46E
+DHIncJf9WBR1dMYyOT+W0L/IsuArmijXnwWNx5p6KhK81nV/ziTOy/6aD2uRlCPv
+FjcjWlhWIgCsUdYA1K4uTb2oM/i52tqT3h9/Qh3EYQYu+X5UB1lx1FQyUZJ6LsUi
+jRHjvITtRAsmZ7l2uP6Ms1lYMfOcaHziKFrbsmppm6eVP1FWG3LEkLxK7/ydZEwa
+SSRenqB4VZJ3ISG/0znidOhntq1AC1UnAazR53CTvDE357aSa49bj9MQADJWNOaB
+mKsEWh4Sg0OOOpQpaS5W7e0beogUz0QybmPitzwlyhzTM/4=
 -----END CERTIFICATE-----

data/cookbooks/mu-tools/recipes/rsyslog.rb

@@ -32,7 +32,7 @@ if !node['application_attributes']['skip_recipes'].include?('rsyslog')
     if platform_family?("rhel")
       $rsyslog_ssl_ca_path = "/etc/pki/Mu_CA.pem"
       if !platform?("amazon")
-        package "policycoreutils-python"
+        package node['platform_version'].to_i < 6 ? "policycoreutils" : "policycoreutils-python"
         execute "allow rsyslog to meddle with port 10514" do
           command "/usr/sbin/semanage port -a -t syslogd_port_t -p tcp 10514"
           not_if "/usr/sbin/semanage port -l | grep '^syslog.*10514'"

data/modules/mu/cleanup.rb

@@ -61,7 +61,7 @@ module MU
       end
 
 
-      types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection", "Habitat"]
+      types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection", "Habitat", "Folder"]
 
       # Load up our deployment metadata
       if !mommacat.nil?
@@ -160,6 +160,19 @@ module MU
                       end
 
                       if @mommacat.nil? or @mommacat.numKittens(types: [t]) > 0
+                        if @mommacat
+                          found = @mommacat.findLitterMate(type: t, return_all: true, credentials: credset)
+                          flags['known'] ||= []
+                          if found.is_a?(Array)
+                            found.each { |k|
+                              flags['known'] << k.cloud_id
+                            }
+                          elsif found and found.is_a?(Hash)
+                            flags['known'] << found['cloud_id']
+                          elsif found
+                            flags['known'] << found.cloud_id                            
+                          end
+                        end
                         begin
                           resclass = Object.const_get("MU").const_get("Cloud").const_get(t)
                           resclass.cleanup(

data/modules/mu/cloud.rb

@@ -126,10 +126,44 @@ module MU
     class NoSQLDB;
     end
 
+    # Denotes a resource implementation which is missing significant
+    # functionality or is largely untested.
+    ALPHA = "This implementation is **ALPHA** quality. It is experimental, may be missing significant functionality, and has not been widely tested."
+
+    # Denotes a resource implementation which supports most or all key API
+    # functionality and has seen at least some non-trivial testing.
+    BETA = "This implementation is **BETA** quality. It is substantially complete, but may be missing some functionality or have some features which are untested."
+
+    # Denotes a resource implementation which supports all key API functionality
+    # and has been substantially tested on real-world applications.
+    RELEASE = "This implementation is considered **RELEASE** quality. It covers all major API features and has been tested with real-world applications."
+
     # The types of cloud resources we can create, as class objects. Include
     # methods a class implementing this resource type must support to be
     # considered valid.
     @@resource_types = {
+      :Folder => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "folder",
+        :cfg_plural => "folders",
+        :interface => self.const_get("Folder"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => true,
+        :class => generic_class_methods,
+        :instance => generic_instance_methods
+      },
+      :Habitat => {
+        :has_multiples => false,
+        :can_live_in_vpc => false,
+        :cfg_name => "habitat",
+        :cfg_plural => "habitats",
+        :interface => self.const_get("Habitat"),
+        :deps_wait_on_my_creation => true,
+        :waits_on_parent_completion => true,
+        :class => generic_class_methods,
+        :instance => generic_instance_methods + [:groom]
+      },
       :Collection => {
         :has_multiples => false,
         :can_live_in_vpc => false,
@@ -328,28 +362,6 @@ module MU
         :class => generic_class_methods,
         :instance => generic_instance_methods + [:groom]
       },
-      :Habitat => {
-        :has_multiples => false,
-        :can_live_in_vpc => false,
-        :cfg_name => "habitat",
-        :cfg_plural => "habitats",
-        :interface => self.const_get("Habitat"),
-        :deps_wait_on_my_creation => true,
-        :waits_on_parent_completion => true,
-        :class => generic_class_methods,
-        :instance => generic_instance_methods
-      },
-      :Folder => {
-        :has_multiples => false,
-        :can_live_in_vpc => false,
-        :cfg_name => "folder",
-        :cfg_plural => "folders",
-        :interface => self.const_get("Folder"),
-        :deps_wait_on_my_creation => true,
-        :waits_on_parent_completion => true,
-        :class => generic_class_methods,
-        :instance => generic_instance_methods
-      },
       :User => {
         :has_multiples => false,
         :can_live_in_vpc => false,
@@ -1028,6 +1040,12 @@ module MU
           return [@dependencies, @vpc, @loadbalancers]
         end
 
+        # Defaults any resources that don't declare their release-readiness to
+        # ALPHA. That'll learn 'em.
+        def self.quality
+          MU::Cloud::ALPHA
+        end
+
         def self.find(*flags)
           allfound = {}
 

data/modules/mu/clouds/aws/alarm.rb

@@ -150,6 +150,12 @@ module MU
           end
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Locate an existing alarm.
         # @param cloud_id [String]: The cloud provider's identifier for this resource.
         # @param region [String]: The cloud provider region.

data/modules/mu/clouds/aws/bucket.rb

@@ -39,6 +39,8 @@ module MU
         # Called automatically by {MU::Deploy#createResources}
         def create
           bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
+
+          MU.log "Creating S3 bucket #{bucket_name}"
           MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
             acl: @config['acl'],
             bucket: bucket_name
@@ -83,6 +85,7 @@ module MU
 
         # Called automatically by {MU::Deploy#createResources}
         def groom
+
           @@region_cache_semaphore.synchronize {
             @@region_cache[@cloud_id] ||= @config['region']
           }
@@ -90,6 +93,17 @@ module MU
 
           current = cloud_desc
 
+          if @config['policies']
+            policy_docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
+            policy_docs.each { |doc|
+              MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first
+              MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
+                bucket: @cloud_id,
+                policy: JSON.generate(doc.values.first)
+              )
+            }
+          end
+
           if @config['web'] and current["website"].nil?
             MU.log "Enabling web service on S3 bucket #{@cloud_id}", MU::NOTICE
             MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_website(
@@ -138,6 +152,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all buckets associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
@@ -217,6 +237,7 @@ module MU
         def self.schema(config)
           toplevel_required = []
           schema = {
+            "policies" => MU::Cloud::AWS::Role.condition_schema,
             "acl" => {
               "type" => "string",
               "enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
@@ -239,6 +260,14 @@ module MU
         def self.validateConfig(bucket, configurator)
           ok = true
 
+          if bucket['policies']
+            bucket['policies'].each { |pol|
+              if !pol['permissions'] or pol['permissions'].empty?
+                pol['permissions'] = ["s3:GetObject"]
+              end
+            }
+          end
+
           ok
         end
 

data/modules/mu/clouds/aws/cache_cluster.rb

@@ -579,6 +579,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the currently-loaded deployment and purges them.
         # @param noop [Boolean]: If true, will only print what would be done.
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server.

data/modules/mu/clouds/aws/container_cluster.rb

@@ -363,6 +363,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all container_clusters associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/database.rb

@@ -1197,6 +1197,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         # @param noop [Boolean]: If true, will only print what would be done

data/modules/mu/clouds/aws/dnszone.rb

@@ -664,6 +664,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Called by {MU::Cleanup}. Locates resources that were created by the
         # currently-loaded deployment, and purges them.
         def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})

data/modules/mu/clouds/aws/endpoint.rb

@@ -448,6 +448,12 @@ MU::Cloud::AWS.apig(region: @config['region'], credentials: @config['credentials
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Canonical Amazon Resource Number for this resource
         # @return [String]
         def arn

data/modules/mu/clouds/aws/firewall_rule.rb

@@ -235,6 +235,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/folder.rb

@@ -55,6 +55,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::ALPHA
+        end
+
         # Remove all logs associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/function.rb

@@ -267,6 +267,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all functions associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/group.rb

@@ -130,6 +130,12 @@ module MU
           true
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::BETA
+        end
+
         # Remove all groups associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server

data/modules/mu/clouds/aws/loadbalancer.rb

@@ -632,6 +632,12 @@ module MU
           false
         end
 
+        # Denote whether this resource implementation is experiment, ready for
+        # testing, or ready for production use.
+        def self.quality
+          MU::Cloud::RELEASE
+        end
+
         # Remove all load balancers associated with the currently loaded deployment.
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server