cloud-mu 2.1.0beta → 3.0.0beta

data/modules/mu/config/bucket.rb

@@ -52,7 +52,7 @@ module MU
             },
             "policies" => {
               "type" => "array",
-              "items" => MU::Config::Role.policy_primitive(subobjects: true, grant_to: true, permissions_optional: true)
+              "items" => MU::Config::Role.policy_primitive(subobjects: true, grant_to: true, permissions_optional: true, targets_optional: true)
             }
           }
         }

data/modules/mu/config/cache_cluster.rb

@@ -163,7 +163,7 @@ module MU
         end
         cluster["multi_az"] = true if cluster["node_count"] > 1
 
-        if !cluster['scrub_mu_isms']
+        if !cluster['scrub_mu_isms'] and cluster["cloud"] != "Azure"
           cluster['dependencies'] << configurator.adminFirewallRuleset(vpc: cluster['vpc'], region: cluster['region'], cloud: cluster['cloud'], credentials: cluster['credentials'])
         end
 

data/modules/mu/config/cache_cluster.yml

@@ -1,22 +1,18 @@
 <% if $complexity == "complex" %>
 name: redis
-credentials: egtprod
 engine: redis
 creation_style: new
 size: cache.t2.medium
 name: memcache
-credentials: egtprod
 creation_style: new
 engine: memcached
 size: cache.t2.medium
 <% else %>
 name: redis
-credentials: egtprod
 engine: redis
 creation_style: new
 size: cache.t2.medium
 name: memcache
-credentials: egtprod
 creation_style: new
 engine: memcached
 size: cache.t2.medium

data/modules/mu/config/container_cluster.rb

@@ -23,8 +23,7 @@ module MU
         base = {
           "type" => "object",
           "description" => "Create a cluster of container hosts.",
-          "required" => ["name", "cloud", "instance_type", "instance_count"],
-          "additionalProperties" => false,
+          "required" => ["name", "cloud", "instance_type"],
           "properties" => {
             "name" => { "type" => "string" },
             "region" => MU::Config.region_primitive,
@@ -35,18 +34,26 @@ module MU
               "type" => "integer",
               "default" => 2
             },
+            "min_size" => {
+              "type" => "integer",
+              "description" => "Enable worker cluster scaling and set the minimum number of workers to this value. This value is ignored for platforms which abstract scaling activity, such as AWS Fargate."
+            },
+            "max_size" => {
+              "type" => "integer",
+              "description" => "Enable worker cluster scaling and set the maximum number of workers to this value. This value is ignored for platforms which abstract scaling activity, such as AWS Fargate."
+            },
             "kubernetes" => {
               "type" => "object",
-              "description" => "Options for Kubernetes, specific to EKS or GKE",
+              "description" => "Kubernetes-specific options",
               "properties" => {
                 "version" => {
                   "type" => "string",
-                  "default" => "1.11",
+                  "default" => "1.13",
                   "description" => "Version of Kubernetes control plane to deploy",
                 },
                 "max_pods" => {
                   "type" => "integer",
-                  "default" => 5,
+                  "default" => 30,
                   "description" => "Maximum number of pods that can be deployed on any given worker node",
                 }
               }
@@ -58,10 +65,6 @@ module MU
                 "description" => "Optional Kubernetes-specific resource descriptors to run with kubectl create|replace when grooming this cluster. See https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#understanding-kubernetes-objects"
               }
             },
-            "flavor" => {
-              "type" => "string",
-              "description" => "Container clusters in Amazon can be ECS, EKS, or Fargate; Google supports GKE only"
-            },
             "platform" => {
               "type" => "string",
               "default" => "linux",
@@ -95,6 +98,12 @@ module MU
       # @return [Boolean]: True if validation succeeded, False otherwise
       def self.validate(cluster, configurator)
         ok = true
+
+        if cluster["max_size"] or cluster["min_size"]
+          cluster["max_size"] ||= [cluster["instance_count"], cluster["min_size"]].reject { |c| c.nil? }.max
+          cluster["min_size"] ||= [cluster["instance_count"], cluster["min_size"]].reject { |c| c.nil? }.min
+        end
+
         ok
       end
 

data/modules/mu/config/database.rb

@@ -42,13 +42,13 @@ module MU
             "tags" => MU::Config.tags_primitive,
             "optional_tags" => MU::Config.optional_tags_primitive,
             "alarms" => MU::Config::Alarm.inline,
-            "engine_version" => {"type" => "string"},
             "add_firewall_rules" => MU::Config::FirewallRule.reference,
             "read_replica_of" => reference,
             "ingress_rules" => {
               "type" => "array",
               "items" => MU::Config::FirewallRule.ruleschema
             },
+            "engine_version" => {"type" => "string"},
             "engine" => {
                 "enum" => ["mysql", "postgres", "oracle-se1", "oracle-se2", "oracle-se", "oracle-ee", "sqlserver-ee", "sqlserver-se", "sqlserver-ex", "sqlserver-web", "aurora", "mariadb"],
                 "type" => "string"
@@ -169,13 +169,7 @@ module MU
             "cluster_node_count" => {
               "type" => "integer",
               "description" => "The number of database instances to add to a database cluster. This only applies to aurora",
-              "default_if" => [
-                {
-                  "key_is" => "engine",
-                  "value_is" => "aurora",
-                  "set" => 1
-                }
-              ]
+              "default" => 2
             },
             "create_cluster" => {
               "type" => "boolean",
@@ -188,17 +182,6 @@ module MU
                   }
                 ]
             },
-            "parameter_group_family" => {
-                "type" => "String",
-                "enum" => [
-                  "postgres9.6", "postgres9.5", "postgres9.4", "postgres9.3", 
-                  "mysql5.1", "mysql5.5", "mysql5.6", "mysql5.7", 
-                  "oracle-ee-11.2", "oracle-ee-12.1", "oracle-se-11.2", "oracle-se-12.1", "oracle-se1-11.2", "oracle-se1-12.1",
-                  "sqlserver-ee-10.5", "sqlserver-ee-11.0", "sqlserver-ee-12.0", "sqlserver-ex-10.5", "sqlserver-ex-11.0", "sqlserver-ex-12.0", "sqlserver-se-10.5", "sqlserver-se-11.0", "sqlserver-se-12.0", "sqlserver-web-10.5", "sqlserver-web-11.0", "sqlserver-web-12.0", 
-                  "aurora5.6", "mariadb-10.0", "mariadb-10.1"
-                ],
-                "description" => "The database family to create the DB Parameter Group for. The family type must be the same type as the database major version - eg if you set engine_version to 9.4.4 the db_family must be set to postgres9.4."
-            },
             "auth_vault" => {
                 "type" => "object",
                 "additionalProperties" => false,
@@ -327,9 +310,9 @@ module MU
 
         if !db["vpc"].nil?
           if db["vpc"]["subnet_pref"] and !db["vpc"]["subnets"]
-            if db["vpc"]["subnet_pref"] = "public"
+            if db["vpc"]["subnet_pref"] == "public"
               db["vpc"]["subnet_pref"] = "all_public"
-            elsif db["vpc"]["subnet_pref"] = "private"
+            elsif db["vpc"]["subnet_pref"] == "private"
               db["vpc"]["subnet_pref"] = "all_private"
             elsif %w{all any}.include? db["vpc"]["subnet_pref"]
               MU.log "subnet_pref #{db["vpc"]["subnet_pref"]} is not supported for database instance.", MU::ERR
@@ -452,8 +435,8 @@ module MU
         end
         db['dependencies'].uniq!
 
-        read_replicas.each { |replica|
-          ok = false if !configurator.insertKitten(replica, "databases")
+        read_replicas.each { |new_replica|
+          ok = false if !configurator.insertKitten(new_replica, "databases")
         }
         cluster_nodes.each { |member|
           ok = false if !configurator.insertKitten(member, "databases")

data/modules/mu/config/firewall_rule.rb

@@ -48,8 +48,8 @@ module MU
               "default" => false
             },
             "rules" => {
-                "type" => "array",
-                "items" => ruleschema
+              "type" => "array",
+              "items" => ruleschema
             }
           }
         }
@@ -61,7 +61,7 @@ module MU
         {
           "type" => "object",
           "description" => "Network ingress and/or egress rules.",
-          "additionalProperties" => false,
+#          "additionalProperties" => false, # inline ingress_rules can have cloud-specific attributes, and this trips those up
           "properties" => {
             "port_range" => {"type" => "string"},
             "port" => {"type" => "integer"},
@@ -71,8 +71,7 @@ module MU
               "type" => "string"
             },
             "ingress" => {
-              "type" => "boolean",
-              "default" => true
+              "type" => "boolean"
             },
             "egress" => {
               "type" => "boolean",
@@ -93,18 +92,13 @@ module MU
       # Schema block for other resources to use when referencing a sibling FirewallRule
       # @return [Hash]
       def self.reference
+        schema_aliases = [
+          { "rule_id" => "id" },
+          { "rule_name" => "name" }
+        ]
         {
           "type" => "array",
-          "items" => {
-            "type" => "object",
-            "additionalProperties" => false,
-            "description" => "Apply one or more network rulesets, defined in this stack or pre-existing, to this resource. Note that if you add a pre-existing ACL to your resource, they must be compatible (e.g. if using VPCs, they must reside in the same VPC).",
-            "minProperties" => 1,
-            "properties" => {
-                "rule_id" => {"type" => "string"},
-                "rule_name" => {"type" => "string"}
-            }
-          }
+          "items" => MU::Config::Ref.schema(schema_aliases, type: "firewall_rules")
         }
       end
 

data/modules/mu/config/folder.rb

@@ -34,27 +34,28 @@ module MU
       # Chunk of schema to reference a folder/OU, here to be embedded
       # into the schemas of other resources.
       def self.reference
-        {
-          "type" => "object",
-          "description" => "Deploy into or connect with resources in a specific account/project",
-          "minProperties" => 1,
-          "additionalProperties" => false,
-          "properties" => {
-            "id" => {
-              "type" => "string",
-              "description" => "Discover this folder/OU by looking by its cloud provider identifier "
-            },
-            "name" => {
-              "type" => "string",
-              "description" => "Discover this folder/OU by Mu-internal name; typically the shorthand 'name' field of an Folder object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
-            },
-            "cloud" => MU::Config.cloud_primitive,
-            "deploy_id" => {
-              "type" => "string",
-              "description" => "Search for this folder in an existing Mu deploy; specify a Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
-            }
-          }
-        }
+#        {
+#          "type" => "object",
+#          "description" => "Deploy into or connect with resources in a specific account/project",
+#          "minProperties" => 1,
+#          "additionalProperties" => false,
+#          "properties" => {
+#            "id" => {
+#              "type" => "string",
+#              "description" => "Discover this folder/OU by looking by its cloud provider identifier "
+#            },
+#            "name" => {
+#              "type" => "string",
+#              "description" => "Discover this folder/OU by Mu-internal name; typically the shorthand 'name' field of an Folder object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
+#            },
+#            "cloud" => MU::Config.cloud_primitive,
+#            "deploy_id" => {
+#              "type" => "string",
+#              "description" => "Search for this folder in an existing Mu deploy; specify a Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
+#            }
+#          }
+#        }
+        MU::Config::Ref.schema(type: "folders")
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::folder}, bare and unvalidated.

data/modules/mu/config/habitat.rb

@@ -34,27 +34,28 @@ module MU
       # Chunk of schema to reference an account/project, here to be embedded
       # into the schemas of other resources.
       def self.reference
-        {
-          "type" => "object",
-          "description" => "Deploy into or connect with resources in a specific habitat (AWS account, GCP project, etc)",
-          "minProperties" => 1,
-          "additionalProperties" => false,
-          "properties" => {
-            "id" => {
-              "type" => "string",
-              "description" => "Discover this habitat by looking for this cloud provider identifier, such as 836541910896 (an AWS account number) or my-project-196124 (a Google Cloud project id)"
-            },
-            "name" => {
-              "type" => "string",
-              "description" => "Discover this habitat by Mu-internal name; typically the shorthand 'name' field of a Habitat object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
-            },
-            "cloud" => MU::Config.cloud_primitive,
-            "deploy_id" => {
-              "type" => "string",
-              "description" => "Search for this Habitat in an existing Mu deploy by Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
-            }
-          }
-        }
+#        {
+#          "type" => "object",
+#          "description" => "Deploy into or connect with resources in a specific habitat (AWS account, GCP project, etc)",
+#          "minProperties" => 1,
+#          "additionalProperties" => false,
+#          "properties" => {
+#            "id" => {
+#              "type" => "string",
+#              "description" => "Discover this habitat by looking for this cloud provider identifier, such as 836541910896 (an AWS account number) or my-project-196124 (a Google Cloud project id)"
+#            },
+#            "name" => {
+#              "type" => "string",
+#              "description" => "Discover this habitat by Mu-internal name; typically the shorthand 'name' field of a Habitat object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
+#            },
+#            "cloud" => MU::Config.cloud_primitive,
+#            "deploy_id" => {
+#              "type" => "string",
+#              "description" => "Search for this Habitat in an existing Mu deploy by Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
+#            }
+#          }
+#        }
+        MU::Config::Ref.schema(type: "habitats")
       end
 
       # Generic pre-processing of {MU::Config::BasketofKittens::habitat}, bare and unvalidated.

data/modules/mu/config/loadbalancer.rb

@@ -103,8 +103,8 @@ module MU
             },
             "alarms" => MU::Config::Alarm.inline,
             "ingress_rules" => {
-                "type" => "array",
-                "items" => MU::Config::FirewallRule.ruleschema
+              "type" => "array",
+              "items" => MU::Config::FirewallRule.ruleschema
             },
             "region" => MU::Config.region_primitive,
             "cross_zone_unstickiness" => {

data/modules/mu/config/role.rb

@@ -48,27 +48,7 @@ module MU
       # Chunk of schema to reference an account/project, here to be embedded
       # into the schemas of other resources.
       def self.reference
-        {
-          "type" => "object",
-          "description" => "An IAM role to associate with this resource",
-          "minProperties" => 1,
-          "additionalProperties" => false,
-          "properties" => {
-            "id" => {
-              "type" => "string",
-              "description" => "Discover this role by looking for this cloud provider identifier, such as an AWS ARN"
-            },
-            "name" => {
-              "type" => "string",
-              "description" => "Discover this role by Mu-internal name; typically the shorthand 'name' field of a Role object declared elsewhere in the deploy, or in another deploy that's being referenced with 'deploy_id'."
-            },
-            "cloud" => MU::Config.cloud_primitive,
-            "deploy_id" => {
-              "type" => "string",
-              "description" => "Search for this Role in an existing Mu deploy by Mu deploy id (e.g. DEMO-DEV-2014111400-NG)."
-            }
-          }
-        }
+        MU::Config::Ref.schema(type: "roles")
       end
 
       # A generic, cloud-neutral descriptor for a policy that grants or denies
@@ -76,11 +56,11 @@ module MU
       # @param subobjects [Boolean]: Whether the returned schema should include a +path+ parameter
       # @param grant_to [Boolean]: Whether the returned schema should include an explicit +grant_to+ parameter
       # @return [Hash]
-      def self.policy_primitive(subobjects: false, grant_to: false, permissions_optional: false)
+      def self.policy_primitive(subobjects: false, grant_to: false, permissions_optional: false, targets_optional: false)
         cfg = {
           "type" => "object",
           "description" => "Policies which grant or deny permissions.",
-          "required" => ["name", "targets"],
+          "required" => ["name"],
 #          "additionalProperties" => false,
           "properties" => {
             "name" => {
@@ -126,28 +106,17 @@ module MU
         }
 
         cfg["required"] << "permissions" if !permissions_optional
+        cfg["required"] << "targets" if !targets_optional
+
+        schema_aliases = [
+          { "identifier" => "id" },
+        ]
 
         if grant_to
           cfg["properties"]["grant_to"] = {
             "type" => "array",
             "default" => [ { "identifier" => "*" } ],
-            "items" => {
-              "type" => "object",
-              "description" => "Entities to which this policy will grant or deny access.",
-              "required" => ["identifier"],
-              "additionalProperties" => false,
-              "properties" => {
-                "type" => {
-                  "type" => "string",
-                  "description" => "A Mu resource type, used when referencing a sibling Mu resource in this stack with +identifier+.",
-                  "enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_name] }.sort
-                },
-                "identifier" => {
-                  "type" => "string",
-                  "description" => "Either the name of a sibling Mu resource in this stack (used in conjunction with +entity_type+), or the full cloud identifier for a resource, such as an Amazon ARN or email-address-formatted Google Cloud username. Wildcards (+*+) are valid if supported by the cloud provider."
-                }
-              }
-            }
+            "items" => MU::Config::Ref.schema(schema_aliases, desc: "Entities to which this policy will grant or deny access.")
           }
         end
 

data/modules/mu/config/server.rb

@@ -120,6 +120,10 @@ module MU
       def self.common_properties
         {
           "name" => {"type" => "string"},
+          "ansible_vars" => {
+            "type" => "object",
+            "description" => "When using Ansible as a groomer, this will insert a +vars+ tree into the playbook for this node."
+          },
           "scrub_mu_isms" => {
               "type" => "boolean",
               "default" => false,
@@ -412,9 +416,8 @@ module MU
           "platform" => {
               "type" => "string",
               "default" => "linux",
-              "enum" => ["linux", "windows", "centos", "ubuntu", "centos6", "ubuntu14", "win2k12", "win2k12r2", "win2k16", "centos7", "rhel7", "rhel71", "amazon"],
-# XXX change to reflect available keys in mu/defaults/amazon_images.yaml and mu/defaults/google_images.yaml
-              "description" => "Helps select default AMIs, and enables correct grooming behavior based on operating system type.",
+              "enum" => MU::Cloud.listPlatforms,
+              "description" => "Helps select default machine images, and enables correct grooming behavior based on operating system type.",
           },
           "run_list" => {
               "type" => "array",
@@ -500,11 +503,19 @@ module MU
           "description" => "Create individual server instances.",
           "properties" => {
               "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+              "bastion" => {
+                "type" => "boolean",
+                "default" => false,
+                "description" => "Allow this server to be automatically used as a bastion host"
+              },
+              "image_id" => {
+                "type" => "string",
+                "description" => "The cloud provider image on which to base this instance. Will use the default appropriate for the +platform+, if not specified."
+              },
               "create_image" => {
                   "type" => "object",
                   "title" => "create_image",
                   "required" => ["image_then_destroy", "image_exclude_storage", "public"],
-                  "additionalProperties" => false,
                   "description" => "Create a reusable image of this server once it is complete.",
                   "properties" => {
                       "public" => {
@@ -570,7 +581,7 @@ module MU
         server['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
         ok = false if !MU::Config.check_vault_refs(server)
 
-        if !server['scrub_mu_isms']
+        if server["cloud"] != "Azure"
           server['dependencies'] << configurator.adminFirewallRuleset(vpc: server['vpc'], region: server['region'], cloud: server['cloud'], credentials: server['credentials'])
         end
 
@@ -593,6 +604,16 @@ module MU
               "name" => configurator.nat_routes[server["vpc"]["subnet_name"]],
               "phase" => "groom"
             }
+          elsif !server["vpc"]["name"].nil?
+            siblingvpc = configurator.haveLitterMate?(server["vpc"]["name"], "vpcs")
+            if siblingvpc and siblingvpc['bastion'] and
+               server['name'] != siblingvpc['bastion'].to_h['name']
+              server["dependencies"] << {
+                "type" => "server",
+                "name" => siblingvpc['bastion'].to_h['name'],
+                "phase" => "groom"
+              }
+            end
           end
         end