cloud-mu 3.1.6 → 3.2.0

data/modules/mu/{clouds → providers}/google/vpc.rb

@@ -539,7 +539,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @return [void]
         def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
           flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -565,7 +565,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['habitat'], credentials: credentials)
+                    fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
                     fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
@@ -946,6 +946,41 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
         end
 
+        # Looks at existing subnets, and attempts to find the next available
+        # IP block that's roughly similar to the ones we already have. This
+        # checks against secondary IP ranges, as well as each subnet's primary
+        # CIDR block.
+        # @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
+        # @return [String]
+        def getUnusedAddressBlock(exclude: [], max_bits: 28)
+          used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
+          subnets.each { |s|
+            used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
+            if s.cloud_desc.secondary_ip_ranges
+              used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
+            end
+          }
+# XXX sort used_ranges
+          candidate = used_ranges.first.next_sib
+
+          begin
+            if candidate.netmask.prefix_len > max_bits
+              candidate = candidate.resize(max_bits)
+            end
+            try_again = false
+            used_ranges.each { |cidr|
+              if !cidr.rel(candidate).nil?
+                candidate = candidate.next_sib
+                try_again = true
+                break
+              end
+            }
+            try_again = false if candidate.nil?
+          end while try_again
+
+          candidate.to_s
+        end
+
         private
 
         def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)

data/modules/tests/centos6.yaml

@@ -0,0 +1,11 @@
+# groomers: Chef
+---
+appname: smoketest
+servers: 
+- name: centos6
+  platform: centos6
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos7.yaml

@@ -0,0 +1,11 @@
+# groomers: Chef
+---
+appname: smoketest
+servers: 
+- name: centos7
+  platform: centos7
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos8.yaml

@@ -0,0 +1,12 @@
+# groomers: Chef
+# clouds: Azure, Google
+---
+appname: smoketest
+servers: 
+- name: centos8
+  platform: centos8
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/rds.yaml

@@ -0,0 +1,108 @@
+# clouds: AWS
+---
+appname: smoketest
+vpcs:
+- name: rdstests
+databases:
+- name: pgcluster
+  size: db.t3.medium
+  engine: postgres
+  engine_version: "10"  
+  auto_minor_version_upgrade: true
+  backup_retention_period: 10
+  cluster_node_count: 2
+  create_cluster: true
+  cluster_parameter_group_parameters:
+  - name: log_disconnections
+    value: "0"
+  vpc:
+    name: rdstests
+  master_user: Bob
+
+#- name: mysqlcluster
+#  size: db.t3.medium
+#  engine: aurora
+#  cluster_mode: serverless
+#  create_cluster: true
+#  vpc:
+#    name: rdstests
+
+- name: maria-base
+  size: db.t2.small
+  engine: mariadb
+  db_parameter_group_parameters:
+  - name: autocommit
+    value: "0"
+  vpc:
+    name: rdstests
+  region: us-east-1
+  create_read_replica: true
+  allow_major_version_upgrade: true
+  read_replica_region: us-east-2
+  cloudwatch_logs:
+  - slowquery
+  multi_az_on_create: true
+  master_user: Bob
+- name: maria-from-snap
+  size: db.t2.small
+  engine: mariadb
+  vpc:
+    name: rdstests
+  creation_style: new_snapshot
+  source:
+    name: maria-base
+- name: maria-point-in-time
+  creation_style: point_in_time
+  size: db.t2.micro
+  engine: mariadb
+  cloudwatch_logs:
+  - error
+  - general
+  source:
+    name: maria-base
+  vpc:
+    name: rdstests
+
+- name: oracle-base
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+- name: oracle-from-snap
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+  creation_style: new_snapshot
+  source:
+    name: oracle-base
+- name: oracle-point-in-time
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+  creation_style: point_in_time
+  source:
+    name: oracle-base
+
+- name: sqlserver-base
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+- name: sqlserver-from-snap
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+  creation_style: new_snapshot
+  source:
+    name: sqlserver-base
+- name: sqlserver-point-in-time
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+  creation_style: point_in_time
+  source:
+    name: sqlserver-base

data/modules/tests/regrooms/rds.yaml

@@ -0,0 +1,123 @@
+# clouds: AWS
+---
+appname: smoketest
+vpcs:
+- name: rdstests
+firewall_rules:
+- name: world
+  vpc:
+    name: rdstests
+  rules:
+  - port: 3307
+    hosts:
+    - 0.0.0.0/0
+databases:
+- name: pgcluster
+  size: db.t3.medium
+  engine: postgres
+  engine_version: "10"  
+  allow_major_version_upgrade: true
+  auto_minor_version_upgrade: false
+  backup_retention_period: 8
+  cluster_node_count: 2
+  create_cluster: true
+  cluster_parameter_group_parameters:
+  - name: log_disconnections
+    value: "0"
+  - name: authentication_timeout
+    value: "35"
+  vpc:
+    name: rdstests
+  master_user: Jimmy
+
+#- name: mysqlcluster
+#  size: db.t3.medium
+#  engine: aurora
+#  cluster_mode: serverless
+#  create_cluster: true
+#  vpc:
+#    name: rdstests
+
+- name: maria-base
+  size: db.t2.small
+  engine: mariadb
+  db_parameter_group_parameters:
+  - name: autocommit
+    value: "1"
+  vpc:
+    name: rdstests
+  region: us-east-1
+  create_read_replica: true
+  read_replica_region: us-east-2
+  cloudwatch_logs:
+  - slowquery
+  - error
+  multi_az_on_create: true
+  master_user: Stoki
+- name: maria-from-snap
+  size: db.t2.small
+  engine: mariadb
+  port: 3307
+  vpc:
+    name: rdstests
+  add_firewall_rules:
+  - name: world 
+  creation_style: new_snapshot
+  source:
+    name: maria-base
+- name: maria-point-in-time
+  creation_style: point_in_time
+  size: db.t2.micro
+  engine: mariadb
+  cloudwatch_logs:
+  - error
+  - general
+  source:
+    name: maria-base
+  vpc:
+    name: rdstests
+
+- name: oracle-base
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+  master_user: helen
+- name: oracle-from-snap
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+  creation_style: new_snapshot
+  source:
+    name: oracle-base
+- name: oracle-point-in-time
+  size: db.m5.large
+  engine: oracle
+  vpc:
+    name: rdstests
+  creation_style: point_in_time
+  source:
+    name: oracle-base
+
+- name: sqlserver-base
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+- name: sqlserver-from-snap
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+  creation_style: new_snapshot
+  source:
+    name: sqlserver-base
+- name: sqlserver-point-in-time
+  size: db.t2.small
+  engine: sqlserver-ex
+  vpc:
+    name: rdstests
+  creation_style: point_in_time
+  source:
+    name: sqlserver-base

data/spec/mu/clouds/azure_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 require 'yaml'
-require 'mu/clouds/azure'
+require 'mu/providers/azure'
 
 
 describe MU::Cloud::Azure do
@@ -216,4 +216,4 @@ describe MU::Cloud::Azure do
 
 	end
 
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-mu
 version: !ruby/object:Gem::Version
-  version: 3.1.6
+  version: 3.2.0
 platform: ruby
 authors:
 - John Stange
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-20 00:00:00.000000000 Z
+date: 2020-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -438,26 +438,28 @@ description: |+
 
 email: eGTLabs@eglobaltech.com
 executables:
-- mu-firewall-allow-clients
-- mu-tunnel-nagios
-- mu-gen-env
-- mu-self-update
-- mu-ssh
-- mu-upload-chef-artifacts
-- mu-configure
-- mu-aws-setup
 - mu-gcp-setup
 - mu-load-config.rb
+- mu-self-update
 - mu-node-manage
-- mu-ansible-secret
-- mu-momma-cat
+- mu-findstray-tests
+- mu-aws-setup
 - mu-azure-setup
 - mu-gen-docs
-- mu-user-manage
-- mu-cleanup
-- mu-deploy
+- mu-tunnel-nagios
+- mu-ssh
+- mu-gen-env
+- mu-configure
+- mu-momma-cat
+- mu-ansible-secret
 - mu-run-tests
+- mu-firewall-allow-clients
+- mu-deploy
+- mu-user-manage
 - mu-adopt
+- mu-upload-chef-artifacts
+- mu-azure-tests
+- mu-cleanup
 extensions: []
 extra_rdoc_files: []
 files:
@@ -510,9 +512,11 @@ files:
 - bin/mu-ansible-secret
 - bin/mu-aws-setup
 - bin/mu-azure-setup
+- bin/mu-azure-tests
 - bin/mu-cleanup
 - bin/mu-configure
 - bin/mu-deploy
+- bin/mu-findstray-tests
 - bin/mu-firewall-allow-clients
 - bin/mu-gcp-setup
 - bin/mu-gen-docs
@@ -952,80 +956,16 @@ files:
 - modules/mu/adoption.rb
 - modules/mu/cleanup.rb
 - modules/mu/cloud.rb
-- modules/mu/clouds/README.md
-- modules/mu/clouds/aws.rb
-- modules/mu/clouds/aws/alarm.rb
-- modules/mu/clouds/aws/bucket.rb
-- modules/mu/clouds/aws/cache_cluster.rb
-- modules/mu/clouds/aws/collection.rb
-- modules/mu/clouds/aws/container_cluster.rb
-- modules/mu/clouds/aws/database.rb
-- modules/mu/clouds/aws/dnszone.rb
-- modules/mu/clouds/aws/endpoint.rb
-- modules/mu/clouds/aws/firewall_rule.rb
-- modules/mu/clouds/aws/folder.rb
-- modules/mu/clouds/aws/function.rb
-- modules/mu/clouds/aws/group.rb
-- modules/mu/clouds/aws/habitat.rb
-- modules/mu/clouds/aws/loadbalancer.rb
-- modules/mu/clouds/aws/log.rb
-- modules/mu/clouds/aws/msg_queue.rb
-- modules/mu/clouds/aws/nosqldb.rb
-- modules/mu/clouds/aws/notifier.rb
-- modules/mu/clouds/aws/role.rb
-- modules/mu/clouds/aws/search_domain.rb
-- modules/mu/clouds/aws/server.rb
-- modules/mu/clouds/aws/server_pool.rb
-- modules/mu/clouds/aws/storage_pool.rb
-- modules/mu/clouds/aws/user.rb
-- modules/mu/clouds/aws/userdata/README.md
-- modules/mu/clouds/aws/userdata/linux.erb
-- modules/mu/clouds/aws/userdata/windows.erb
-- modules/mu/clouds/aws/vpc.rb
-- modules/mu/clouds/aws/vpc_subnet.rb
-- modules/mu/clouds/azure.rb
-- modules/mu/clouds/azure/container_cluster.rb
-- modules/mu/clouds/azure/firewall_rule.rb
-- modules/mu/clouds/azure/habitat.rb
-- modules/mu/clouds/azure/loadbalancer.rb
-- modules/mu/clouds/azure/role.rb
-- modules/mu/clouds/azure/server.rb
-- modules/mu/clouds/azure/user.rb
-- modules/mu/clouds/azure/userdata/README.md
-- modules/mu/clouds/azure/userdata/linux.erb
-- modules/mu/clouds/azure/userdata/windows.erb
-- modules/mu/clouds/azure/vpc.rb
-- modules/mu/clouds/cloudformation.rb
-- modules/mu/clouds/cloudformation/alarm.rb
-- modules/mu/clouds/cloudformation/cache_cluster.rb
-- modules/mu/clouds/cloudformation/collection.rb
-- modules/mu/clouds/cloudformation/database.rb
-- modules/mu/clouds/cloudformation/dnszone.rb
-- modules/mu/clouds/cloudformation/firewall_rule.rb
-- modules/mu/clouds/cloudformation/loadbalancer.rb
-- modules/mu/clouds/cloudformation/log.rb
-- modules/mu/clouds/cloudformation/server.rb
-- modules/mu/clouds/cloudformation/server_pool.rb
-- modules/mu/clouds/cloudformation/vpc.rb
-- modules/mu/clouds/docker.rb
-- modules/mu/clouds/google.rb
-- modules/mu/clouds/google/bucket.rb
-- modules/mu/clouds/google/container_cluster.rb
-- modules/mu/clouds/google/database.rb
-- modules/mu/clouds/google/firewall_rule.rb
-- modules/mu/clouds/google/folder.rb
-- modules/mu/clouds/google/function.rb
-- modules/mu/clouds/google/group.rb
-- modules/mu/clouds/google/habitat.rb
-- modules/mu/clouds/google/loadbalancer.rb
-- modules/mu/clouds/google/role.rb
-- modules/mu/clouds/google/server.rb
-- modules/mu/clouds/google/server_pool.rb
-- modules/mu/clouds/google/user.rb
-- modules/mu/clouds/google/userdata/README.md
-- modules/mu/clouds/google/userdata/linux.erb
-- modules/mu/clouds/google/userdata/windows.erb
-- modules/mu/clouds/google/vpc.rb
+- modules/mu/cloud/database.rb
+- modules/mu/cloud/dnszone.rb
+- modules/mu/cloud/machine_images.rb
+- modules/mu/cloud/providers.rb
+- modules/mu/cloud/resource_base.rb
+- modules/mu/cloud/server.rb
+- modules/mu/cloud/server_pool.rb
+- modules/mu/cloud/ssh_sessions.rb
+- modules/mu/cloud/winrm_sessions.rb
+- modules/mu/cloud/wrappers.rb
 - modules/mu/config.rb
 - modules/mu/config/README.md
 - modules/mu/config/alarm.rb
@@ -1091,12 +1031,89 @@ files:
 - modules/mu/mommacat/search.rb
 - modules/mu/mommacat/storage.rb
 - modules/mu/monkey_patches/chef_knife_ssh.rb
+- modules/mu/providers/README.md
+- modules/mu/providers/aws.rb
+- modules/mu/providers/aws/alarm.rb
+- modules/mu/providers/aws/bucket.rb
+- modules/mu/providers/aws/cache_cluster.rb
+- modules/mu/providers/aws/collection.rb
+- modules/mu/providers/aws/container_cluster.rb
+- modules/mu/providers/aws/database.rb
+- modules/mu/providers/aws/dnszone.rb
+- modules/mu/providers/aws/endpoint.rb
+- modules/mu/providers/aws/firewall_rule.rb
+- modules/mu/providers/aws/folder.rb
+- modules/mu/providers/aws/function.rb
+- modules/mu/providers/aws/group.rb
+- modules/mu/providers/aws/habitat.rb
+- modules/mu/providers/aws/loadbalancer.rb
+- modules/mu/providers/aws/log.rb
+- modules/mu/providers/aws/msg_queue.rb
+- modules/mu/providers/aws/nosqldb.rb
+- modules/mu/providers/aws/notifier.rb
+- modules/mu/providers/aws/role.rb
+- modules/mu/providers/aws/search_domain.rb
+- modules/mu/providers/aws/server.rb
+- modules/mu/providers/aws/server_pool.rb
+- modules/mu/providers/aws/storage_pool.rb
+- modules/mu/providers/aws/user.rb
+- modules/mu/providers/aws/userdata/README.md
+- modules/mu/providers/aws/userdata/linux.erb
+- modules/mu/providers/aws/userdata/windows.erb
+- modules/mu/providers/aws/vpc.rb
+- modules/mu/providers/aws/vpc_subnet.rb
+- modules/mu/providers/azure.rb
+- modules/mu/providers/azure/container_cluster.rb
+- modules/mu/providers/azure/firewall_rule.rb
+- modules/mu/providers/azure/habitat.rb
+- modules/mu/providers/azure/loadbalancer.rb
+- modules/mu/providers/azure/role.rb
+- modules/mu/providers/azure/server.rb
+- modules/mu/providers/azure/user.rb
+- modules/mu/providers/azure/userdata/README.md
+- modules/mu/providers/azure/userdata/linux.erb
+- modules/mu/providers/azure/userdata/windows.erb
+- modules/mu/providers/azure/vpc.rb
+- modules/mu/providers/cloudformation.rb
+- modules/mu/providers/cloudformation/alarm.rb
+- modules/mu/providers/cloudformation/cache_cluster.rb
+- modules/mu/providers/cloudformation/collection.rb
+- modules/mu/providers/cloudformation/database.rb
+- modules/mu/providers/cloudformation/dnszone.rb
+- modules/mu/providers/cloudformation/firewall_rule.rb
+- modules/mu/providers/cloudformation/loadbalancer.rb
+- modules/mu/providers/cloudformation/log.rb
+- modules/mu/providers/cloudformation/server.rb
+- modules/mu/providers/cloudformation/server_pool.rb
+- modules/mu/providers/cloudformation/vpc.rb
+- modules/mu/providers/docker.rb
+- modules/mu/providers/google.rb
+- modules/mu/providers/google/bucket.rb
+- modules/mu/providers/google/container_cluster.rb
+- modules/mu/providers/google/database.rb
+- modules/mu/providers/google/firewall_rule.rb
+- modules/mu/providers/google/folder.rb
+- modules/mu/providers/google/function.rb
+- modules/mu/providers/google/group.rb
+- modules/mu/providers/google/habitat.rb
+- modules/mu/providers/google/loadbalancer.rb
+- modules/mu/providers/google/role.rb
+- modules/mu/providers/google/server.rb
+- modules/mu/providers/google/server_pool.rb
+- modules/mu/providers/google/user.rb
+- modules/mu/providers/google/userdata/README.md
+- modules/mu/providers/google/userdata/linux.erb
+- modules/mu/providers/google/userdata/windows.erb
+- modules/mu/providers/google/vpc.rb
 - modules/tests/README.md
 - modules/tests/application_attributes.inc
 - modules/tests/auto_scaling.inc
 - modules/tests/aws-iam.yaml
 - modules/tests/aws-sgs.yaml
 - modules/tests/bucket.yml
+- modules/tests/centos6.yaml
+- modules/tests/centos7.yaml
+- modules/tests/centos8.yaml
 - modules/tests/dnszone.yaml
 - modules/tests/ecs.yaml
 - modules/tests/efs.yaml
@@ -1109,8 +1126,10 @@ files:
 - modules/tests/log.yaml
 - modules/tests/needwork/google-projects.yaml
 - modules/tests/poolparams-include.inc
+- modules/tests/rds.yaml
 - modules/tests/regrooms/aws-iam.yaml
 - modules/tests/regrooms/bucket.yml
+- modules/tests/regrooms/rds.yaml
 - modules/tests/server-with-scrub-muisms.yaml
 - modules/tests/super_complex_bok.yml
 - modules/tests/super_simple_bok.yml