cloud-mu 1.9.0.pre.beta

data/modules/mu/config/server_pool.rb

@@ -0,0 +1,216 @@
+# Copyright:: Copyright (c) 2018 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/server_pool.rb
+    class ServerPool
+
+      # Base configuration schema for a ServerPool
+      # @return [Hash]
+      def self.schema
+        base = {
+          "type" => "object",
+          "additionalProperties" => false,
+          "description" => "Create scalable pools of identical servers.",
+          "required" => ["name", "min_size", "max_size", "basis", "cloud"],
+          "properties" => {
+            "dns_records" => MU::Config::DNSZone.records_primitive(need_target: false, default_type: "A", need_zone: true),
+            "scrub_mu_isms" => {
+                "type" => "boolean",
+                "default" => false,
+                "description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template."
+            },
+            "wait_for_nodes" => {
+                "type" => "integer",
+                "description" => "Use this parameter to force a certain number of nodes to come up and be fully bootstrapped before the rest of the pool is initialized.",
+                "default" => 0,
+            },
+            "vpc" => MU::Config::VPC.reference(MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NAT_OPTS, "all_private"),
+            "min_size" => {"type" => "integer"},
+            "max_size" => {"type" => "integer"},
+            "tags" => MU::Config.tags_primitive,
+            "optional_tags" => MU::Config.optional_tags_primitive,
+            "desired_capacity" => {
+                "type" => "integer",
+                "description" => "The number of Amazon EC2 instances that should be running in the group. Should be between min_size and max_size."
+            },
+            "default_cooldown" => {
+                "type" => "integer",
+                "default" => 300
+            },
+            "health_check_type" => {
+                "type" => "string",
+                "enum" => ["EC2", "ELB"],
+                "default" => "EC2",
+            },
+            "health_check_grace_period" => {
+                "type" => "integer",
+                "default" => 0
+            },
+            "vpc_zone_identifier" => {
+                "type" => "string",
+                "description" => "A comma-separated list of subnet identifiers of Amazon Virtual Private Clouds (Amazon VPCs).
+
+          If you specify subnets and Availability Zones with this call, ensure that the subnets' Availability Zones match the Availability Zones specified."
+            },
+            #XXX this needs its own primitive and discovery mechanism
+            "zones" => {
+              "type" => "array",
+              "minItems" => 1,
+              "items" => {
+                "type" => "string",
+              }
+            },
+            "basis" => {
+              "type" => "object",
+              "minProperties" => 1,
+              "maxProperties" => 1,
+              "additionalProperties" => false,
+              "description" => "The baseline for new servers created within this Autoscale Group.",
+              "properties" => {
+                "instance_id" => {
+                    "type" => "string",
+                    "description" => "The AWS instance ID of an existing instance to use as the base image for this Autoscale Group.",
+                },
+                "server" => {
+                    "type" => "string",
+                    "description" => "Build a server defined elsewhere in this stack, then use it as the base image for this Autoscale Group.",
+                },
+                "launch_config" => {
+                  "type" => "object",
+                  "required" => ["name", "size"],
+                  "minProperties" => 3,
+                  "additionalProperties" => false,
+                  "description" => "An Amazon Launch Config for an Autoscale Group.",
+                  "properties" => {
+                    "name" => {"type" => "string"},
+                    "instance_id" => {
+                      "type" => "string",
+                      "description" => "The AWS instance ID of an existing instance to use as the base image in this Launch Config.",
+                    },
+                    "storage" => MU::Config::Server.storage_primitive,
+                    "server" => {
+                      "type" => "string",
+                      "description" => "Build a server defined elsewhere in this stack, create an AMI from it, then use it as the base image in this Launch Config.",
+                    },
+                    "ami_id" => {
+                      "type" => "string",
+                      "description" => "The Amazon EC2 AMI to use as the base image in this Launch Config. Will use the default for platform if not specified.",
+                    },
+                    "image_id" => {
+                      "type" => "string",
+                      "description" => "The Google Cloud Platform Image on which to base this autoscaler. Will use the default appropriate for the platform, if not specified.",
+                    },
+                    "monitoring" => {
+                      "type" => "boolean",
+                      "default" => true,
+                      "description" => "Enable instance monitoring?",
+                    },
+                    "ebs_optimized" => {
+                      "type" => "boolean",
+                      "default" => false,
+                      "description" => "EBS optimized?",
+                    },
+                    "iam_role" => {
+                      "type" => "string",
+                      "description" => "An Amazon IAM instance profile, from which to harvest role policies to merge into this node's own instance profile. If generate_iam_role is false, will simple use this profile.",
+                    },
+                    "generate_iam_role" => {
+                      "type" => "boolean",
+                      "default" => true,
+                      "description" => "Generate a unique IAM profile for this Server or ServerPool.",
+                    },
+                    "iam_policies" => {
+                      "type" => "array",
+                      "items" => {
+                        "description" => "Amazon-compatible role policies which will be merged into this node's own instance profile.  Not valid with generate_iam_role set to false. Our parser expects the role policy document to me embedded under a named container, e.g. { 'name_of_policy':'{ <policy document> } }",
+                        "type" => "object"
+                      }
+                    },
+                    "spot_price" => {
+                      "type" => "string",
+                    },
+                    "kernel_id" => {
+                      "type" => "string",
+                      "description" => "Kernel to use with servers created from this Launch Configuration.",
+                    },
+                    "ramdisk_id" => {
+                      "type" => "string",
+                      "description" => "Kernel to use with servers created from this Launch Configuration.",
+                    },
+                    "size" => {
+                      "description" => "The Amazon EC2 instance type to use when creating this server.",
+                      "type" => "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        base["properties"].merge!(MU::Config::Server.common_properties)
+        base
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::server_pools}, bare and unvalidated.
+      # @param pool [Hash]: The resource to process and validate
+      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(pool, configurator)
+        ok = true
+        if configurator.haveLitterMate?(pool["name"], "servers")
+          MU.log "Can't use name #{pool['name']} more than once in pools/pool_pools"
+          ok = false
+        end
+        pool['skipinitialupdates'] = true if configurator.skipinitialupdates
+        pool['ingress_rules'] ||= []
+        pool['vault_access'] ||= []
+        pool['vault_access'] << {"vault" => "splunk", "item" => "admin_user"}
+        ok = false if !MU::Config.check_vault_refs(pool)
+
+        pool['dependencies'] << configurator.adminFirewallRuleset(vpc: pool['vpc'], region: pool['region'], cloud: pool['cloud']) if !pool['scrub_mu_isms']
+
+        if !pool["vpc"].nil?
+          if !pool["vpc"]["subnet_name"].nil? and configurator.nat_routes.has_key?(pool["vpc"]["subnet_name"])
+            pool["dependencies"] << {
+              "type" => "pool",
+              "name" => configurator.nat_routes[pool["vpc"]["subnet_name"]],
+              "phase" => "groom"
+            }
+          end
+        end
+# TODO make sure this is handled... somewhere
+#        if pool["alarms"] && !pool["alarms"].empty?
+#          pool["alarms"].each { |alarm|
+#            alarm["name"] = "server-#{pool['name']}-#{alarm["name"]}"
+#            alarm["namespace"] = "AWS/EC2" if alarm["namespace"].nil?
+#            alarm['cloud'] = pool['cloud']
+#            ok = false if !insertKitten(alarm, "alarms")
+#          }
+#        end
+        if pool["basis"]["server"] != nil
+          pool["dependencies"] << {"type" => "server", "name" => pool["basis"]["server"]}
+        end
+        if !pool['static_ip'].nil? and !pool['ip'].nil?
+          ok = false
+          MU.log "Server Pools cannot assign specific static IPs.", MU::ERR
+        end
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/server_pool.yml

@@ -0,0 +1,71 @@
+<% if $complexity == 'complex' %>
+name: <%= server_pools_name %>
+cloud: AWS
+alarms:
+- comparison_operator: "GreaterThanThreshold"
+  metric_name: "HTTPCode_Target_5XX_Count"
+  evaluation_periods: 1
+  name: "HTTP_Target_500_Count"
+  period: 60
+  statistic: "Sum"
+  threshold: 10
+  enable_notifications: true
+  notification_endpoint: 'admin@example.com'
+  namespace: "AWS/ApplicationELB"
+scale_in_protection: "initial"
+scale_with_cpu: 50.0
+scaling_policies:
+- name: simplediskreads
+  policy_type: SimpleScaling
+  adjustment: 1
+  cooldown: 60
+  type: ChangeInCapacity
+  alarms:
+  - name: simplenetwork
+    namespace: "AWS/EC2"
+    metric_name: DiskReadBytes
+    period: 60
+    evaluation_periods: 1
+    comparison_operator: GreaterThanThreshold
+    statistic: Sum
+    threshold: 1073741824
+- name: trackingnetworkin
+  policy_type: TargetTrackingScaling
+  estimated_instance_warmup: 600
+  target_tracking_configuration:
+    target_value: 250.0
+    predefined_metric_specification: ASGAverageNetworkIn
+termination_policies:
+- "OldestInstance"
+wait_for_nodes: 1
+min_size: 2
+max_size: 2
+schedule:
+- action_name: scale-down-over-night
+  recurrence: "30 6 * * *"
+  min_size: 1
+  max_size: 1
+- action_name: scale-up-during-the-day
+  recurrence: "30 12 * * *"
+  min_size: 2
+  max_size: 2
+ingress_rules:
+- port: 80
+  hosts:
+  - 0.0.0.0/0
+- port: 443
+  hosts:
+  - 0.0.0.0/0
+basis:
+  launch_config:
+    name: <%= server_pools_name %>
+    size: t2.medium
+<% else %>
+name: <%= server_pools_name %>
+min_size: 1
+max_size: 1
+basis:
+  launch_config:
+    name: <%= server_pools_name %>
+    size: t2.small
+<% end %>

data/modules/mu/config/storage_pool.rb

@@ -0,0 +1,145 @@
+# Copyright:: Copyright (c) 2018 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
+    class StoragePool
+
+      # Base configuration schema for a StoragePool
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "title" => "Storage Pool",
+          "description" => "Create a storage pool.",
+          "required" => ["name", "cloud"],
+          "additionalProperties" => false,
+          "properties" => {
+            "cloud" => MU::Config.cloud_primitive,
+            "name" => {"type" => "string"},
+            "region" => MU::Config.region_primitive,
+            "tags" => MU::Config.tags_primitive,
+            "optional_tags" => MU::Config.optional_tags_primitive,
+            "dependencies" => MU::Config.dependencies_primitive,
+            "storage_type" => {
+              "type" => "string",
+              "enum" => ["generalPurpose", "maxIO"],
+              "description" => "The storage type / performance mode of this storage pool. Defaults to generalPurpose",
+              "default" => "generalPurpose"
+            },
+            "mount_points" => {
+              "type" => "array",
+              "minItems" => 1,
+              "items" => {
+                "type" => "object",
+                "required" => ["name"],
+                "additionalProperties" => false,
+                "description" => "Mount points for AWS EFS.",
+                "properties" => {
+                  "name" => {
+                    "type" => "string"
+                  },
+                  "directory" => {
+                    "type" => "string",
+                    "description" => "The local directory this mount point will be mounted to",
+                    "default" => "/efs"
+                  },
+                  "vpc" => MU::Config::VPC.reference(MU::Config::VPC::ONE_SUBNET+MU::Config::VPC::MANY_SUBNETS, MU::Config::VPC::NO_NAT_OPTS, "all_private"),
+                  "add_firewall_rules" => MU::Config::FirewallRule.reference,
+                  "ingress_rules" => {
+                    "type" => "array",
+                    "description" => "Firewall rules to apply to our mountpoints",
+                    "items" => MU::Config::FirewallRule.ruleschema
+                  },
+                  "ip_address" => {
+                    "type" => "string",
+                    "pattern" => "^\\d+\\.\\d+\\.\\d+\\.\\d+$",
+                    "description" => "The private IP address to assign to the mount point."
+                  }
+                }
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::storage_pools}, bare and unvalidated.
+      # @param pool [Hash]: The resource to process and validate
+      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(pool, configurator)
+        ok = true
+        if pool['mount_points']
+          new_mount_points = []
+          pool['mount_points'].each{ |mp|
+            if mp["vpc"] and !mp["vpc"].empty?
+              if !mp["vpc"]["vpc_name"].nil? and
+                 configurator.haveLitterMate?(mp["vpc"]["vpc_name"], "vpcs") and
+                 mp["vpc"]['deploy_id'].nil? and
+                 mp["vpc"]['vpc_id'].nil?
+      
+                siblingvpc = configurator.haveLitterMate?(mp["vpc"]["vpc_name"], "vpcs")
+                if !MU::Config::VPC.processReference(mp['vpc'],
+                                        "storage_pools",
+                                        "storagepool '#{pool['name']}'",
+                                        configurator,
+                                        dflt_region: pool['region'],
+                                        is_sibling: true,
+                                        sibling_vpcs: [siblingvpc])
+                  ok = false
+                end
+              else
+                if !MU::Config::VPC.processReference(mp["vpc"],
+                                        "storage_pools",
+                                        "storagepool #{pool['name']}",
+                                        configurator,
+                                        dflt_region: pool['region'])
+                  ok = false
+                end
+              end
+              if mp['vpc']['subnets'] and mp['vpc']['subnets'].size > 1
+                seen_azs = []
+                count = 0
+                mp['vpc']['subnets'].each { |subnet|
+                  if subnet['az'] and seen_azs.include?(subnet['az'])
+                    MU.log "VPC config for Storage Pool #{pool['name']} has multiple matching subnets per Availability Zone. Only one mount point per AZ is allowed, so you must explicitly declare which subnets to use.", MU::ERR
+                    ok = false
+                    break
+                  end
+                  seen_azs << subnet['az']
+                  subnet.delete("az")
+                  newmp = Marshal.load(Marshal.dump(mp))
+                  ["subnets", "subnet_pref", "az"].each { |field|
+                    newmp['vpc'].delete(field)
+                  }
+                  newmp['vpc'].merge!(subnet)
+                  newmp['name'] = newmp['name']+count.to_s
+                  count = count + 1
+                  new_mount_points << newmp
+                }
+              else
+                new_mount_points << mp
+              end
+            end
+          }
+          pool['mount_points'] = new_mount_points
+        end
+
+        ok
+      end
+
+    end
+  end
+end

data/modules/mu/config/user.rb

@@ -0,0 +1,78 @@
+# Copyright:: Copyright (c) 2018 eGlobalTech, Inc., all rights reserved
+#
+# Licensed under the BSD-3 license (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License in the root of the project or at
+#
+#     http://egt-labs.com/mu/LICENSE.html
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module MU
+  class Config
+    # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/user.rb
+    class User
+
+      # Base configuration schema for a User
+      # @return [Hash]
+      def self.schema
+        {
+          "type" => "object",
+          "additionalProperties" => false,
+          "description" => "Set up a cloud provider user or machine account",
+          "required" => ["name", "type"],
+          "properties" => {
+            "name" => {
+              "type" => "string",
+              "description" => "The name of the account to create or associate."
+            },
+            "type" => {
+              "type" => "string",
+              "description" => "Indicates whether to create or associate an account meant for interactive human use, or for a machine or service.",
+              "enum" => ["interactive", "service"],
+              "default" => "interactive"
+            },
+            "use_if_exists" => {
+              "type" => "boolean",
+              "description" => "If we attempt to create or associate a user that already exists, simply modify that user in-place and use it, rather than throwing an error. If this flag is set, the user will *not* be deleted on cleanup, nor will we overwrite any existing tags on cloud platforms that support user tagging.",
+              "default" => true
+            },
+            "create_api_key" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Create a set of cloud API keys for this user. Keys will be shared via Scratchpad for one-time retrieval."
+            },
+            "preserve_on_cleanup" => {
+              "type" => "boolean",
+              "default" => false,
+              "description" => "Leave the user intact during the cleanup process. If we are re-using an existing user, rather than creating one ourselves, this option has no effect- that user will always be left intact."
+            },
+            "groups" => {
+              "type" => "array",
+              "description" => "One or more groups to associate with this user.",
+              "items" => {
+                "type" => "string",
+                "description" => "One or more groups to associate with this user. If there is a 'group' resource defined with this name in this Basket of Kittens, we will use that; if not, and if there is an existing cloud provider group in the appropriate account/project that matches, we will use that; if neither of those exists, we will implicitly create a matching group if it had been declared in this Basket of Kittens."
+              }
+            }
+          }
+        }
+      end
+
+      # Generic pre-processing of {MU::Config::BasketofKittens::user}, bare and unvalidated.
+      # @param user [Hash]: The resource to process and validate
+      # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
+      # @return [Boolean]: True if validation succeeded, False otherwise
+      def self.validate(user, configurator)
+        ok = true
+
+        ok
+      end
+
+    end
+  end
+end