cloud-mu 2.1.0beta → 3.0.0beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Berksfile +4 -5
- data/Berksfile.lock +179 -0
- data/README.md +1 -6
- data/ansible/roles/geerlingguy.firewall/templates/firewall.bash.j2 +0 -0
- data/ansible/roles/mu-installer/README.md +33 -0
- data/ansible/roles/mu-installer/defaults/main.yml +2 -0
- data/ansible/roles/mu-installer/handlers/main.yml +2 -0
- data/ansible/roles/mu-installer/meta/main.yml +60 -0
- data/ansible/roles/mu-installer/tasks/main.yml +13 -0
- data/ansible/roles/mu-installer/tests/inventory +2 -0
- data/ansible/roles/mu-installer/tests/test.yml +5 -0
- data/ansible/roles/mu-installer/vars/main.yml +2 -0
- data/bin/mu-adopt +125 -0
- data/bin/mu-aws-setup +4 -4
- data/bin/mu-azure-setup +265 -0
- data/bin/mu-azure-tests +43 -0
- data/bin/mu-cleanup +20 -8
- data/bin/mu-configure +224 -98
- data/bin/mu-deploy +8 -3
- data/bin/mu-gcp-setup +16 -8
- data/bin/mu-gen-docs +92 -8
- data/bin/mu-load-config.rb +52 -12
- data/bin/mu-momma-cat +36 -0
- data/bin/mu-node-manage +34 -27
- data/bin/mu-self-update +2 -2
- data/bin/mu-ssh +12 -8
- data/bin/mu-upload-chef-artifacts +11 -4
- data/bin/mu-user-manage +3 -0
- data/cloud-mu.gemspec +8 -11
- data/cookbooks/firewall/libraries/helpers_iptables.rb +2 -2
- data/cookbooks/firewall/metadata.json +1 -1
- data/cookbooks/firewall/recipes/default.rb +5 -9
- data/cookbooks/mu-firewall/attributes/default.rb +2 -0
- data/cookbooks/mu-firewall/metadata.rb +1 -1
- data/cookbooks/mu-glusterfs/templates/default/mu-gluster-client.erb +0 -0
- data/cookbooks/mu-master/Berksfile +2 -2
- data/cookbooks/mu-master/files/default/check_mem.pl +0 -0
- data/cookbooks/mu-master/files/default/cloudamatic.png +0 -0
- data/cookbooks/mu-master/metadata.rb +5 -4
- data/cookbooks/mu-master/recipes/389ds.rb +1 -1
- data/cookbooks/mu-master/recipes/basepackages.rb +30 -10
- data/cookbooks/mu-master/recipes/default.rb +59 -7
- data/cookbooks/mu-master/recipes/firewall-holes.rb +1 -1
- data/cookbooks/mu-master/recipes/init.rb +65 -47
- data/cookbooks/mu-master/recipes/{eks-kubectl.rb → kubectl.rb} +4 -10
- data/cookbooks/mu-master/recipes/sssd.rb +2 -1
- data/cookbooks/mu-master/recipes/update_nagios_only.rb +6 -6
- data/cookbooks/mu-master/templates/default/web_app.conf.erb +2 -2
- data/cookbooks/mu-master/templates/mods/ldap.conf.erb +4 -0
- data/cookbooks/mu-php54/Berksfile +1 -2
- data/cookbooks/mu-php54/metadata.rb +4 -5
- data/cookbooks/mu-php54/recipes/default.rb +1 -1
- data/cookbooks/mu-splunk/templates/default/splunk-init.erb +0 -0
- data/cookbooks/mu-tools/Berksfile +3 -2
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +33 -0
- data/cookbooks/mu-tools/libraries/helper.rb +20 -8
- data/cookbooks/mu-tools/metadata.rb +5 -2
- data/cookbooks/mu-tools/recipes/apply_security.rb +2 -3
- data/cookbooks/mu-tools/recipes/eks.rb +1 -1
- data/cookbooks/mu-tools/recipes/gcloud.rb +5 -30
- data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
- data/cookbooks/mu-tools/recipes/rsyslog.rb +1 -0
- data/cookbooks/mu-tools/recipes/selinux.rb +19 -0
- data/cookbooks/mu-tools/recipes/split_var_partitions.rb +0 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +256 -122
- data/cookbooks/mu-tools/resources/disk.rb +3 -1
- data/cookbooks/mu-tools/templates/amazon/sshd_config.erb +1 -1
- data/cookbooks/mu-tools/templates/default/etc_hosts.erb +1 -1
- data/cookbooks/mu-tools/templates/default/{kubeconfig.erb → kubeconfig-eks.erb} +0 -0
- data/cookbooks/mu-tools/templates/default/kubeconfig-gke.erb +27 -0
- data/cookbooks/mu-tools/templates/windows-10/sshd_config.erb +137 -0
- data/cookbooks/mu-utility/recipes/nat.rb +4 -0
- data/extras/alpha.png +0 -0
- data/extras/beta.png +0 -0
- data/extras/clean-stock-amis +2 -2
- data/extras/generate-stock-images +131 -0
- data/extras/git-fix-permissions-hook +0 -0
- data/extras/image-generators/AWS/centos6.yaml +17 -0
- data/extras/image-generators/{aws → AWS}/centos7-govcloud.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/centos7.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/rhel7.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/win2k12.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/win2k16.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/windows.yaml +0 -0
- data/extras/image-generators/{gcp → Google}/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +18 -0
- data/extras/python_rpm/build.sh +0 -0
- data/extras/release.png +0 -0
- data/extras/ruby_rpm/build.sh +0 -0
- data/extras/ruby_rpm/muby.spec +1 -1
- data/install/README.md +43 -5
- data/install/deprecated-bash-library.sh +0 -0
- data/install/installer +1 -1
- data/install/jenkinskeys.rb +0 -0
- data/install/mu-master.yaml +55 -0
- data/modules/mommacat.ru +41 -7
- data/modules/mu.rb +444 -149
- data/modules/mu/adoption.rb +500 -0
- data/modules/mu/cleanup.rb +235 -158
- data/modules/mu/cloud.rb +675 -138
- data/modules/mu/clouds/aws.rb +156 -24
- data/modules/mu/clouds/aws/alarm.rb +4 -14
- data/modules/mu/clouds/aws/bucket.rb +60 -18
- data/modules/mu/clouds/aws/cache_cluster.rb +8 -20
- data/modules/mu/clouds/aws/collection.rb +12 -22
- data/modules/mu/clouds/aws/container_cluster.rb +209 -118
- data/modules/mu/clouds/aws/database.rb +120 -45
- data/modules/mu/clouds/aws/dnszone.rb +7 -18
- data/modules/mu/clouds/aws/endpoint.rb +5 -15
- data/modules/mu/clouds/aws/firewall_rule.rb +144 -72
- data/modules/mu/clouds/aws/folder.rb +4 -11
- data/modules/mu/clouds/aws/function.rb +6 -16
- data/modules/mu/clouds/aws/group.rb +4 -12
- data/modules/mu/clouds/aws/habitat.rb +11 -13
- data/modules/mu/clouds/aws/loadbalancer.rb +40 -28
- data/modules/mu/clouds/aws/log.rb +5 -13
- data/modules/mu/clouds/aws/msg_queue.rb +9 -24
- data/modules/mu/clouds/aws/nosqldb.rb +4 -12
- data/modules/mu/clouds/aws/notifier.rb +6 -13
- data/modules/mu/clouds/aws/role.rb +69 -40
- data/modules/mu/clouds/aws/search_domain.rb +17 -20
- data/modules/mu/clouds/aws/server.rb +184 -94
- data/modules/mu/clouds/aws/server_pool.rb +33 -38
- data/modules/mu/clouds/aws/storage_pool.rb +5 -12
- data/modules/mu/clouds/aws/user.rb +59 -33
- data/modules/mu/clouds/aws/userdata/linux.erb +18 -30
- data/modules/mu/clouds/aws/userdata/windows.erb +9 -9
- data/modules/mu/clouds/aws/vpc.rb +214 -145
- data/modules/mu/clouds/azure.rb +978 -44
- data/modules/mu/clouds/azure/container_cluster.rb +413 -0
- data/modules/mu/clouds/azure/firewall_rule.rb +500 -0
- data/modules/mu/clouds/azure/habitat.rb +167 -0
- data/modules/mu/clouds/azure/loadbalancer.rb +205 -0
- data/modules/mu/clouds/azure/role.rb +211 -0
- data/modules/mu/clouds/azure/server.rb +810 -0
- data/modules/mu/clouds/azure/user.rb +257 -0
- data/modules/mu/clouds/azure/userdata/README.md +4 -0
- data/modules/mu/clouds/azure/userdata/linux.erb +137 -0
- data/modules/mu/clouds/azure/userdata/windows.erb +275 -0
- data/modules/mu/clouds/azure/vpc.rb +782 -0
- data/modules/mu/clouds/cloudformation.rb +12 -9
- data/modules/mu/clouds/cloudformation/firewall_rule.rb +5 -13
- data/modules/mu/clouds/cloudformation/server.rb +10 -1
- data/modules/mu/clouds/cloudformation/server_pool.rb +1 -0
- data/modules/mu/clouds/cloudformation/vpc.rb +0 -2
- data/modules/mu/clouds/google.rb +554 -117
- data/modules/mu/clouds/google/bucket.rb +173 -32
- data/modules/mu/clouds/google/container_cluster.rb +1112 -157
- data/modules/mu/clouds/google/database.rb +24 -47
- data/modules/mu/clouds/google/firewall_rule.rb +344 -89
- data/modules/mu/clouds/google/folder.rb +156 -79
- data/modules/mu/clouds/google/group.rb +272 -82
- data/modules/mu/clouds/google/habitat.rb +177 -52
- data/modules/mu/clouds/google/loadbalancer.rb +9 -34
- data/modules/mu/clouds/google/role.rb +1211 -0
- data/modules/mu/clouds/google/server.rb +491 -227
- data/modules/mu/clouds/google/server_pool.rb +233 -48
- data/modules/mu/clouds/google/user.rb +479 -125
- data/modules/mu/clouds/google/userdata/linux.erb +3 -3
- data/modules/mu/clouds/google/userdata/windows.erb +9 -9
- data/modules/mu/clouds/google/vpc.rb +381 -223
- data/modules/mu/config.rb +689 -214
- data/modules/mu/config/bucket.rb +1 -1
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/cache_cluster.yml +0 -4
- data/modules/mu/config/container_cluster.rb +18 -9
- data/modules/mu/config/database.rb +6 -23
- data/modules/mu/config/firewall_rule.rb +9 -15
- data/modules/mu/config/folder.rb +22 -21
- data/modules/mu/config/habitat.rb +22 -21
- data/modules/mu/config/loadbalancer.rb +2 -2
- data/modules/mu/config/role.rb +9 -40
- data/modules/mu/config/server.rb +26 -5
- data/modules/mu/config/server_pool.rb +1 -1
- data/modules/mu/config/storage_pool.rb +2 -2
- data/modules/mu/config/user.rb +4 -0
- data/modules/mu/config/vpc.rb +350 -110
- data/modules/mu/defaults/{amazon_images.yaml → AWS.yaml} +37 -39
- data/modules/mu/defaults/Azure.yaml +17 -0
- data/modules/mu/defaults/Google.yaml +24 -0
- data/modules/mu/defaults/README.md +1 -1
- data/modules/mu/deploy.rb +168 -125
- data/modules/mu/groomer.rb +2 -1
- data/modules/mu/groomers/ansible.rb +104 -32
- data/modules/mu/groomers/chef.rb +96 -44
- data/modules/mu/kittens.rb +20602 -0
- data/modules/mu/logger.rb +38 -11
- data/modules/mu/master.rb +90 -8
- data/modules/mu/master/chef.rb +2 -3
- data/modules/mu/master/ldap.rb +0 -1
- data/modules/mu/master/ssl.rb +250 -0
- data/modules/mu/mommacat.rb +917 -513
- data/modules/scratchpad.erb +1 -1
- data/modules/tests/super_complex_bok.yml +0 -0
- data/modules/tests/super_simple_bok.yml +0 -0
- data/roles/mu-master.json +2 -1
- data/spec/azure_creds +5 -0
- data/spec/mu.yaml +56 -0
- data/spec/mu/clouds/azure_spec.rb +164 -27
- data/spec/spec_helper.rb +5 -0
- data/test/clean_up.py +0 -0
- data/test/exec_inspec.py +0 -0
- data/test/exec_mu_install.py +0 -0
- data/test/exec_retry.py +0 -0
- data/test/smoke_test.rb +0 -0
- metadata +90 -118
- data/cookbooks/mu-jenkins/Berksfile +0 -14
- data/cookbooks/mu-jenkins/CHANGELOG.md +0 -13
- data/cookbooks/mu-jenkins/LICENSE +0 -37
- data/cookbooks/mu-jenkins/README.md +0 -105
- data/cookbooks/mu-jenkins/attributes/default.rb +0 -42
- data/cookbooks/mu-jenkins/files/default/cleanup_deploy_config.xml +0 -73
- data/cookbooks/mu-jenkins/files/default/deploy_config.xml +0 -44
- data/cookbooks/mu-jenkins/metadata.rb +0 -21
- data/cookbooks/mu-jenkins/recipes/default.rb +0 -195
- data/cookbooks/mu-jenkins/recipes/node-ssh-config.rb +0 -54
- data/cookbooks/mu-jenkins/recipes/public_key.rb +0 -24
- data/cookbooks/mu-jenkins/templates/default/example_job.config.xml.erb +0 -24
- data/cookbooks/mu-jenkins/templates/default/org.jvnet.hudson.plugins.SSHBuildWrapper.xml.erb +0 -14
- data/cookbooks/mu-jenkins/templates/default/ssh_config.erb +0 -6
- data/cookbooks/nagios/Berksfile +0 -11
- data/cookbooks/nagios/CHANGELOG.md +0 -589
- data/cookbooks/nagios/CONTRIBUTING.md +0 -11
- data/cookbooks/nagios/LICENSE +0 -37
- data/cookbooks/nagios/README.md +0 -328
- data/cookbooks/nagios/TESTING.md +0 -2
- data/cookbooks/nagios/attributes/config.rb +0 -171
- data/cookbooks/nagios/attributes/default.rb +0 -228
- data/cookbooks/nagios/chefignore +0 -102
- data/cookbooks/nagios/definitions/command.rb +0 -33
- data/cookbooks/nagios/definitions/contact.rb +0 -33
- data/cookbooks/nagios/definitions/contactgroup.rb +0 -33
- data/cookbooks/nagios/definitions/host.rb +0 -33
- data/cookbooks/nagios/definitions/hostdependency.rb +0 -33
- data/cookbooks/nagios/definitions/hostescalation.rb +0 -34
- data/cookbooks/nagios/definitions/hostgroup.rb +0 -33
- data/cookbooks/nagios/definitions/nagios_conf.rb +0 -38
- data/cookbooks/nagios/definitions/resource.rb +0 -33
- data/cookbooks/nagios/definitions/service.rb +0 -33
- data/cookbooks/nagios/definitions/servicedependency.rb +0 -33
- data/cookbooks/nagios/definitions/serviceescalation.rb +0 -34
- data/cookbooks/nagios/definitions/servicegroup.rb +0 -33
- data/cookbooks/nagios/definitions/timeperiod.rb +0 -33
- data/cookbooks/nagios/libraries/base.rb +0 -314
- data/cookbooks/nagios/libraries/command.rb +0 -91
- data/cookbooks/nagios/libraries/contact.rb +0 -230
- data/cookbooks/nagios/libraries/contactgroup.rb +0 -112
- data/cookbooks/nagios/libraries/custom_option.rb +0 -36
- data/cookbooks/nagios/libraries/data_bag_helper.rb +0 -23
- data/cookbooks/nagios/libraries/default.rb +0 -90
- data/cookbooks/nagios/libraries/host.rb +0 -412
- data/cookbooks/nagios/libraries/hostdependency.rb +0 -181
- data/cookbooks/nagios/libraries/hostescalation.rb +0 -173
- data/cookbooks/nagios/libraries/hostgroup.rb +0 -119
- data/cookbooks/nagios/libraries/nagios.rb +0 -282
- data/cookbooks/nagios/libraries/resource.rb +0 -59
- data/cookbooks/nagios/libraries/service.rb +0 -455
- data/cookbooks/nagios/libraries/servicedependency.rb +0 -215
- data/cookbooks/nagios/libraries/serviceescalation.rb +0 -195
- data/cookbooks/nagios/libraries/servicegroup.rb +0 -144
- data/cookbooks/nagios/libraries/timeperiod.rb +0 -160
- data/cookbooks/nagios/libraries/users_helper.rb +0 -54
- data/cookbooks/nagios/metadata.rb +0 -25
- data/cookbooks/nagios/recipes/_load_databag_config.rb +0 -153
- data/cookbooks/nagios/recipes/_load_default_config.rb +0 -241
- data/cookbooks/nagios/recipes/apache.rb +0 -48
- data/cookbooks/nagios/recipes/default.rb +0 -204
- data/cookbooks/nagios/recipes/nginx.rb +0 -82
- data/cookbooks/nagios/recipes/pagerduty.rb +0 -143
- data/cookbooks/nagios/recipes/server_package.rb +0 -40
- data/cookbooks/nagios/recipes/server_source.rb +0 -164
- data/cookbooks/nagios/templates/default/apache2.conf.erb +0 -96
- data/cookbooks/nagios/templates/default/cgi.cfg.erb +0 -266
- data/cookbooks/nagios/templates/default/commands.cfg.erb +0 -13
- data/cookbooks/nagios/templates/default/contacts.cfg.erb +0 -37
- data/cookbooks/nagios/templates/default/hostgroups.cfg.erb +0 -25
- data/cookbooks/nagios/templates/default/hosts.cfg.erb +0 -15
- data/cookbooks/nagios/templates/default/htpasswd.users.erb +0 -6
- data/cookbooks/nagios/templates/default/nagios.cfg.erb +0 -22
- data/cookbooks/nagios/templates/default/nginx.conf.erb +0 -62
- data/cookbooks/nagios/templates/default/pagerduty.cgi.erb +0 -185
- data/cookbooks/nagios/templates/default/resource.cfg.erb +0 -27
- data/cookbooks/nagios/templates/default/servicedependencies.cfg.erb +0 -15
- data/cookbooks/nagios/templates/default/servicegroups.cfg.erb +0 -14
- data/cookbooks/nagios/templates/default/services.cfg.erb +0 -14
- data/cookbooks/nagios/templates/default/templates.cfg.erb +0 -31
- data/cookbooks/nagios/templates/default/timeperiods.cfg.erb +0 -13
- data/extras/image-generators/aws/centos6.yaml +0 -18
- data/modules/mu/defaults/google_images.yaml +0 -16
- data/roles/mu-master-jenkins.json +0 -24
@@ -104,7 +104,7 @@ if [ ! -f /opt/chef/embedded/bin/ruby ];then
|
|
104
104
|
set +e
|
105
105
|
# We may run afoul of a synchronous bootstrap process doing the same thing. So
|
106
106
|
# wait until we've managed to run successfully.
|
107
|
-
while ! sh chef-install.sh -v <%=
|
107
|
+
while ! sh chef-install.sh -v <%= $mu.chefVersion %>;do
|
108
108
|
sleep 10
|
109
109
|
done
|
110
110
|
touch /opt/mu_installed_chef
|
@@ -117,7 +117,7 @@ if [ "$need_reboot" == "1" ];then
|
|
117
117
|
fi
|
118
118
|
<% end %>
|
119
119
|
|
120
|
-
gsutil cp gs://<%=
|
120
|
+
gsutil cp gs://<%= $mu.adminBucketName %>/<%= $mu.muID %>-secret .
|
121
121
|
|
122
122
|
echo '
|
123
123
|
require "openssl"
|
@@ -132,6 +132,6 @@ instance_id="`curl http://metadata.google.internal/computeMetadata/v1/instance/n
|
|
132
132
|
# Make double-sure sshd is actually up
|
133
133
|
service sshd restart
|
134
134
|
|
135
|
-
/usr/bin/curl -k --data mu_id="<%= $mu.muID %>" --data mu_resource_name="<%= $mu.resourceName %>" --data mu_resource_type="<%= $mu.resourceType %>" --data mu_instance_id="$instance_id" --data mu_bootstrap="1" --data mu_user="<%= $mu.muUser %>" --data mu_deploy_secret="`/opt/chef/embedded/bin/ruby encrypt_deploy_secret.rb`" https://<%= $mu.publicIP
|
135
|
+
/usr/bin/curl -k --data mu_id="<%= $mu.muID %>" --data mu_resource_name="<%= $mu.resourceName %>" --data mu_resource_type="<%= $mu.resourceType %>" --data mu_instance_id="$instance_id" --data mu_bootstrap="1" --data mu_user="<%= $mu.muUser %>" --data mu_deploy_secret="`/opt/chef/embedded/bin/ruby encrypt_deploy_secret.rb`" https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %>/
|
136
136
|
/bin/rm -f <%= $mu.muID %>-secret mu_deploy_key.pub chef-install.sh encrypt_deploy_secret.rb
|
137
137
|
touch /.mu_userdata_complete
|
@@ -22,8 +22,8 @@ function log
|
|
22
22
|
}
|
23
23
|
|
24
24
|
function fetchSecret([string]$file){
|
25
|
-
log "Fetching s3://<%=
|
26
|
-
aws.cmd s3 cp s3://<%=
|
25
|
+
log "Fetching s3://<%= $mu.adminBucketName %>/$file to $tmp/$file"
|
26
|
+
aws.cmd s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file
|
27
27
|
}
|
28
28
|
|
29
29
|
function importCert([string]$cert, [string]$store){
|
@@ -112,7 +112,7 @@ function removeChef($location){
|
|
112
112
|
$install_chef = $false
|
113
113
|
$my_chef = (Get-ItemProperty $location | Where-Object {$_.DisplayName -like "chef client*"}).DisplayName
|
114
114
|
if ($my_chef) {
|
115
|
-
if ($my_chef -match '<%=
|
115
|
+
if ($my_chef -match '<%= $mu.chefVersion %>'.split('-')[0]) {
|
116
116
|
$install_chef = $false
|
117
117
|
} else{
|
118
118
|
log "Uninstalling Chef"
|
@@ -142,13 +142,13 @@ If (!(Test-Path "c:\opscode\chef\embedded\bin\ruby.exe")){
|
|
142
142
|
}
|
143
143
|
|
144
144
|
If ($install_chef){
|
145
|
-
log "Installing Chef <%=
|
146
|
-
If (!(Test-Path $env:Temp/chef-installer-<%=
|
145
|
+
log "Installing Chef <%= $mu.chefVersion %>"
|
146
|
+
If (!(Test-Path $env:Temp/chef-installer-<%= $mu.chefVersion %>.msi)){
|
147
147
|
log "Downloading Chef installer"
|
148
|
-
$WebClient.DownloadFile("https://www.chef.io/chef/download?p=windows&pv=2012&m=x86_64&v=<%=
|
148
|
+
$WebClient.DownloadFile("https://www.chef.io/chef/download?p=windows&pv=2012&m=x86_64&v=<%= $mu.chefVersion %>","$env:Temp/chef-installer-<%= $mu.chefVersion %>.msi")
|
149
149
|
}
|
150
150
|
log "Running Chef installer"
|
151
|
-
(Start-Process -FilePath msiexec -ArgumentList "/i $env:Temp\chef-installer-<%=
|
151
|
+
(Start-Process -FilePath msiexec -ArgumentList "/i $env:Temp\chef-installer-<%= $mu.chefVersion %>.msi ALLUSERS=1 /le $env:Temp\chef-client-install.log /qn" -Wait -Passthru).ExitCode
|
152
152
|
Set-Content "c:/mu_installed_chef" "yup"
|
153
153
|
}
|
154
154
|
|
@@ -159,9 +159,9 @@ $deploy_secret = & "c:\opscode\chef\embedded\bin\ruby" -ropenssl -rbase64 -e "ke
|
|
159
159
|
function callMomma([string]$act)
|
160
160
|
{
|
161
161
|
$params = @{mu_id='<%= $mu.muID %>';mu_resource_name='<%= $mu.resourceName %>';mu_resource_type='<%= $mu.resourceType %>';mu_instance_id="$awsid";mu_user='<%= $mu.muUser %>';mu_deploy_secret="$deploy_secret";$act="1"}
|
162
|
-
log "Calling Momma Cat at https://<%= $mu.publicIP
|
162
|
+
log "Calling Momma Cat at https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %> with $act"
|
163
163
|
[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true} # XXX
|
164
|
-
$resp = Invoke-WebRequest -Uri https://<%= $mu.publicIP
|
164
|
+
$resp = Invoke-WebRequest -Uri https://<%= $mu.publicIP %>:<%= $mu.mommaCatPort %> -Method POST -Body $params
|
165
165
|
return $resp.Content
|
166
166
|
}
|
167
167
|
|
@@ -18,51 +18,24 @@ module MU
|
|
18
18
|
|
19
19
|
# Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
|
20
20
|
class VPC < MU::Cloud::VPC
|
21
|
+
attr_reader :cloud_desc_cache
|
22
|
+
attr_reader :routes
|
21
23
|
|
22
|
-
|
23
|
-
@
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
attr_reader :config
|
29
|
-
|
30
|
-
# @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
|
31
|
-
# @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::vpcs}
|
32
|
-
def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
|
33
|
-
@deploy = mommacat
|
34
|
-
@config = MU::Config.manxify(kitten_cfg)
|
35
|
-
@subnets = []
|
24
|
+
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like <tt>@vpc</tt>, for us.
|
25
|
+
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
26
|
+
def initialize(**args)
|
27
|
+
super
|
28
|
+
|
29
|
+
@subnets ||= []
|
36
30
|
@subnetcachesemaphore = Mutex.new
|
37
|
-
if cloud_id and cloud_id.match(/^https:\/\//)
|
38
|
-
@url = cloud_id.clone
|
39
|
-
@cloud_id = cloud_id.to_s.gsub(/.*?\//, "")
|
40
|
-
elsif cloud_id and !cloud_id.empty?
|
41
|
-
@cloud_id = cloud_id.to_s
|
42
|
-
end
|
43
31
|
|
44
|
-
if
|
45
|
-
@mu_name = mu_name
|
46
|
-
if @cloud_id.nil? or @cloud_id.empty?
|
47
|
-
@cloud_id = MU::Cloud::Google.nameStr(@mu_name)
|
48
|
-
end
|
49
|
-
@config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
|
50
|
-
if !@project_id
|
51
|
-
project = MU::Cloud::Google.projectLookup(@config['project'], @deploy, sibling_only: true, raise_on_fail: false)
|
52
|
-
@project_id = project.nil? ? @config['project'] : project.cloudobj.cloud_id
|
53
|
-
end
|
54
|
-
loadSubnets
|
55
|
-
elsif @config['scrub_mu_isms']
|
56
|
-
@mu_name = @config['name']
|
57
|
-
else
|
58
|
-
@mu_name = @deploy.getResourceName(@config['name'])
|
59
|
-
end
|
32
|
+
loadSubnets if @cloud_id
|
60
33
|
|
34
|
+
@mu_name ||= @config['scrub_mu_isms'] ? @config['name'] : @deploy.getResourceName(@config['name'])
|
61
35
|
end
|
62
36
|
|
63
37
|
# Called automatically by {MU::Deploy#createResources}
|
64
38
|
def create
|
65
|
-
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
|
66
39
|
|
67
40
|
networkobj = MU::Cloud::Google.compute(:Network).new(
|
68
41
|
name: MU::Cloud::Google.nameStr(@mu_name),
|
@@ -73,7 +46,7 @@ module MU
|
|
73
46
|
MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@project_id}", details: networkobj
|
74
47
|
|
75
48
|
resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_network(@project_id, networkobj)
|
76
|
-
@url = resp.self_link
|
49
|
+
@url = resp.self_link
|
77
50
|
@cloud_id = resp.name
|
78
51
|
|
79
52
|
if @config['subnets']
|
@@ -82,8 +55,9 @@ module MU
|
|
82
55
|
@config['subnets'].each { |subnet|
|
83
56
|
subnetthreads << Thread.new {
|
84
57
|
MU.dupGlobals(parent_thread_id)
|
85
|
-
subnet_name =
|
86
|
-
|
58
|
+
subnet_name = subnet['name']
|
59
|
+
|
60
|
+
subnet_mu_name = @config['scrub_mu_isms'] ? @cloud_id+subnet_name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet_name, max_length: 61))
|
87
61
|
MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id}", details: subnet
|
88
62
|
subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
|
89
63
|
name: subnet_mu_name,
|
@@ -92,7 +66,14 @@ module MU
|
|
92
66
|
network: @url,
|
93
67
|
region: subnet['availability_zone']
|
94
68
|
)
|
95
|
-
|
69
|
+
MU::Cloud::Google.compute(credentials: @config['credentials']).insert_subnetwork(@project_id, subnet['availability_zone'], subnetobj)
|
70
|
+
|
71
|
+
# make sure the subnet we created exists, before moving on
|
72
|
+
subnetdesc = nil
|
73
|
+
begin
|
74
|
+
subnetdesc = MU::Cloud::Google.compute(credentials: @config['credentials']).get_subnetwork(@project_id, subnet['availability_zone'], subnet_mu_name)
|
75
|
+
sleep 1
|
76
|
+
end while subnetdesc.nil?
|
96
77
|
|
97
78
|
}
|
98
79
|
}
|
@@ -129,36 +110,42 @@ module MU
|
|
129
110
|
def notify
|
130
111
|
base = MU.structToHash(cloud_desc)
|
131
112
|
base["cloud_id"] = @cloud_id
|
132
|
-
base["project_id"] =
|
113
|
+
base["project_id"] = habitat_id
|
133
114
|
base.merge!(@config.to_h)
|
115
|
+
if @subnets
|
116
|
+
base["subnets"] = @subnets.map { |s| s.notify }
|
117
|
+
end
|
134
118
|
base
|
135
119
|
end
|
136
120
|
|
137
121
|
# Describe this VPC from the cloud platform's perspective
|
138
|
-
# @return [
|
122
|
+
# @return [Google::Apis::Core::Hashable]
|
139
123
|
def cloud_desc
|
124
|
+
if @cloud_desc_cache
|
125
|
+
return @cloud_desc_cache
|
126
|
+
end
|
140
127
|
|
141
128
|
resp = MU::Cloud::Google.compute(credentials: @config['credentials']).get_network(@project_id, @cloud_id)
|
142
|
-
|
129
|
+
|
130
|
+
if @cloud_id.nil? or @cloud_id == "" or resp.nil?
|
143
131
|
MU.log "Couldn't describe #{self}, @cloud_id #{@cloud_id.nil? ? "undefined" : "empty" }", MU::ERR
|
144
132
|
return nil
|
145
133
|
end
|
134
|
+
@cloud_desc_cache = resp
|
146
135
|
|
147
|
-
|
148
|
-
@url ||= resp
|
136
|
+
# populate other parts and pieces of ourself
|
137
|
+
@url ||= resp.self_link
|
149
138
|
routes = MU::Cloud::Google.compute(credentials: @config['credentials']).list_routes(
|
150
139
|
@project_id,
|
151
|
-
filter: "network
|
140
|
+
filter: "network = \"#{@url}\""
|
152
141
|
).items
|
153
|
-
|
154
|
-
# XXX subnets too
|
142
|
+
@routes = routes if routes and routes.size > 0
|
155
143
|
|
156
|
-
|
144
|
+
@cloud_desc_cache
|
157
145
|
end
|
158
146
|
|
159
147
|
# Called automatically by {MU::Deploy#createResources}
|
160
148
|
def groom
|
161
|
-
@project_id = MU::Cloud::Google.projectLookup(@config['project'], @deploy).cloudobj.cloud_id
|
162
149
|
|
163
150
|
rtb = @config['route_tables'].first
|
164
151
|
|
@@ -173,26 +160,32 @@ module MU
|
|
173
160
|
if !@config['peers'].nil?
|
174
161
|
count = 0
|
175
162
|
@config['peers'].each { |peer|
|
176
|
-
if peer['vpc']['
|
177
|
-
peer_obj = @deploy.findLitterMate(name: peer['vpc']['
|
163
|
+
if peer['vpc']['name']
|
164
|
+
peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs", habitat: peer['vpc']['project'])
|
178
165
|
else
|
179
166
|
tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
|
180
|
-
if peer['vpc']['deploy_id'].nil? and peer['vpc']['
|
167
|
+
if peer['vpc']['deploy_id'].nil? and peer['vpc']['id'].nil? and tag_key.nil?
|
181
168
|
peer['vpc']['deploy_id'] = @deploy.deploy_id
|
182
169
|
end
|
183
170
|
|
184
171
|
peer_obj = MU::MommaCat.findStray(
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
172
|
+
"Google",
|
173
|
+
"vpcs",
|
174
|
+
deploy_id: peer['vpc']['deploy_id'],
|
175
|
+
cloud_id: peer['vpc']['id'],
|
176
|
+
name: peer['vpc']['name'],
|
177
|
+
# XXX project flag tho
|
178
|
+
tag_key: tag_key,
|
179
|
+
tag_value: tag_value,
|
180
|
+
dummy_ok: true
|
193
181
|
).first
|
194
182
|
end
|
183
|
+
if peer_obj.nil?
|
184
|
+
MU.log "Failed VPC peer lookup on behalf of #{@cloud_id}", MU::WARN, details: peer
|
185
|
+
pr = peer['vpc']['project'] || @project_id
|
186
|
+
MU.log "all the VPCs I can see", MU::WARN, details: MU::Cloud::Google.compute(credentials: @config['credentials']).list_networks(pr)
|
195
187
|
|
188
|
+
end
|
196
189
|
raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil?
|
197
190
|
|
198
191
|
url = if peer_obj.cloudobj.url
|
@@ -200,7 +193,6 @@ module MU
|
|
200
193
|
elsif peer_obj.cloudobj.deploydata
|
201
194
|
peer_obj.cloudobj.deploydata['self_link']
|
202
195
|
else
|
203
|
-
pp peer_obj.cloudobj.cloud_desc
|
204
196
|
raise MuError, "Can't find the damn URL of my damn peer VPC #{peer['vpc']}"
|
205
197
|
end
|
206
198
|
cnxn_name = MU::Cloud::Google.nameStr(@mu_name+"-peer-"+count.to_s)
|
@@ -227,41 +219,47 @@ module MU
|
|
227
219
|
count += 1
|
228
220
|
}
|
229
221
|
end
|
222
|
+
loadSubnets(use_cache: false)
|
230
223
|
end
|
231
224
|
|
232
|
-
# Locate
|
233
|
-
#
|
234
|
-
#
|
235
|
-
#
|
236
|
-
#
|
237
|
-
#
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
225
|
+
# Locate and return cloud provider descriptors of this resource type
|
226
|
+
# which match the provided parameters, or all visible resources if no
|
227
|
+
# filters are specified. At minimum, implementations of +find+ must
|
228
|
+
# honor +credentials+ and +cloud_id+ arguments. We may optionally
|
229
|
+
# support other search methods, such as +tag_key+ and +tag_value+, or
|
230
|
+
# cloud-specific arguments like +project+. See also {MU::MommaCat.findStray}.
|
231
|
+
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
232
|
+
# @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
|
233
|
+
def self.find(**args)
|
234
|
+
args[:project] ||= args[:habitat]
|
235
|
+
args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
|
242
236
|
resp = {}
|
243
|
-
if cloud_id
|
244
|
-
|
245
|
-
|
246
|
-
|
237
|
+
if args[:cloud_id] and args[:project]
|
238
|
+
begin
|
239
|
+
vpc = MU::Cloud::Google.compute(credentials: args[:credentials]).get_network(
|
240
|
+
args[:project],
|
241
|
+
args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1')
|
247
242
|
)
|
248
|
-
resp[cloud_id] = vpc if !vpc.nil?
|
243
|
+
resp[args[:cloud_id]] = vpc if !vpc.nil?
|
244
|
+
rescue ::Google::Apis::ClientError => e
|
245
|
+
MU.log "Do not have permissions to retrieve VPC #{args[:cloud_id]} in project #{args[:project]}", MU::WARN, details: caller
|
246
|
+
end
|
249
247
|
else # XXX other criteria
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
248
|
+
vpcs = begin
|
249
|
+
MU::Cloud::Google.compute(credentials: args[:credentials]).list_networks(
|
250
|
+
args[:project]
|
251
|
+
)
|
252
|
+
rescue ::Google::Apis::ClientError => e
|
253
|
+
raise e if !e.message.match(/^(?:notFound|forbidden): /)
|
254
|
+
end
|
255
|
+
|
256
|
+
if vpcs and vpcs.items
|
257
|
+
vpcs.items.each { |v|
|
258
|
+
resp[vpc.name] = v
|
259
|
+
}
|
260
|
+
end
|
255
261
|
end
|
256
|
-
|
257
|
-
resp.each_pair { |cloud_id, vpc|
|
258
|
-
routes = MU::Cloud::Google.compute(credentials: credentials).list_routes(
|
259
|
-
flags["project"],
|
260
|
-
filter: "network eq #{vpc.self_link}"
|
261
|
-
).items
|
262
|
-
# pp routes
|
263
|
-
}
|
264
|
-
#MU.log "RETURNING RESPONSE FROM VPC FIND (#{resp.class.name})", MU::WARN, details: resp
|
262
|
+
|
265
263
|
resp
|
266
264
|
end
|
267
265
|
|
@@ -279,72 +277,93 @@ module MU
|
|
279
277
|
# Describe subnets associated with this VPC. We'll compose identifying
|
280
278
|
# information similar to what MU::Cloud.describe builds for first-class
|
281
279
|
# resources.
|
280
|
+
# @param use_cache [Boolean]: If available, use saved deployment metadata to describe subnets, instead of querying the cloud API
|
282
281
|
# @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
|
283
|
-
def loadSubnets
|
282
|
+
def loadSubnets(use_cache: true)
|
283
|
+
@subnetcachesemaphore.synchronize {
|
284
|
+
return @subnets if use_cache and @subnets and @subnets.size > 0
|
285
|
+
}
|
284
286
|
network = cloud_desc
|
287
|
+
|
285
288
|
if network.nil?
|
286
289
|
MU.log "Unabled to load cloud description in #{self}", MU::ERR
|
287
290
|
return nil
|
288
291
|
end
|
289
292
|
found = []
|
290
293
|
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
+
if @deploy and @deploy.deployment and
|
295
|
+
@deploy.deployment["vpcs"] and
|
296
|
+
@deploy.deployment["vpcs"][@config['name']] and
|
297
|
+
@deploy.deployment["vpcs"][@config['name']]["subnets"] and
|
298
|
+
@deploy.deployment["vpcs"][@config['name']]["subnets"].size > 0
|
299
|
+
@deploy.deployment["vpcs"][@config['name']]["subnets"].each { |desc|
|
300
|
+
subnet = {}
|
301
|
+
subnet["ip_block"] = desc['ip_block']
|
302
|
+
subnet["name"] = desc["name"]
|
303
|
+
subnet['mu_name'] = @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
|
304
|
+
subnet["cloud_id"] = desc['cloud_id']
|
305
|
+
subnet["cloud_id"] ||= desc['self_link'].gsub(/.*?\/([^\/]+)$/, '\1')
|
306
|
+
subnet["cloud_id"] ||= subnet['mu_name']
|
307
|
+
subnet['az'] = desc["az"]
|
308
|
+
subnet['az'] ||= desc["region"].gsub(/.*?\/([^\/]+)$/, '\1')
|
309
|
+
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
|
310
|
+
}
|
311
|
+
else
|
312
|
+
resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_subnetwork_usable(
|
294
313
|
@project_id,
|
295
|
-
|
296
|
-
filter: "network eq #{network[:self_link]}"
|
314
|
+
filter: "network eq #{network.self_link}"
|
297
315
|
)
|
298
|
-
next if resp.nil? or resp.items.nil?
|
299
316
|
resp.items.each { |subnet|
|
300
317
|
found << subnet
|
301
318
|
}
|
302
|
-
}
|
303
319
|
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
+
@subnetcachesemaphore.synchronize {
|
321
|
+
@subnets ||= []
|
322
|
+
ext_ids = @subnets.each.collect { |s| s.cloud_id }
|
323
|
+
# If we're a plain old Mu resource, load our config and deployment
|
324
|
+
# metadata. Like ya do.
|
325
|
+
if !@config.nil? and @config.has_key?("subnets")
|
326
|
+
@config['subnets'].each { |subnet|
|
327
|
+
# subnet['mu_name'] = @mu_name+"-"+subnet['name'] if !subnet.has_key?("mu_name")
|
328
|
+
subnet['mu_name'] ||= @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
|
329
|
+
subnet['region'] = @config['region']
|
330
|
+
found.each { |desc|
|
331
|
+
if desc.ip_cidr_range == subnet["ip_block"]
|
332
|
+
desc.subnetwork.match(/\/projects\/[^\/]+\/regions\/([^\/]+)\/subnetworks\/(.+)$/)
|
333
|
+
subnet['az'] = Regexp.last_match[1]
|
334
|
+
subnet['name'] ||= Regexp.last_match[2]
|
335
|
+
subnet["cloud_id"] = subnet['mu_name']
|
336
|
+
subnet["url"] = desc.subnetwork
|
337
|
+
break
|
338
|
+
end
|
339
|
+
}
|
340
|
+
|
341
|
+
if !ext_ids.include?(subnet["cloud_id"])
|
342
|
+
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
|
320
343
|
end
|
321
344
|
}
|
322
345
|
|
346
|
+
# Of course we might be loading up a dummy subnet object from a
|
347
|
+
# foreign or non-Mu-created VPC and subnet. So make something up.
|
348
|
+
elsif !found.nil?
|
349
|
+
found.each { |desc|
|
350
|
+
subnet = {}
|
351
|
+
desc.subnetwork.match(/\/projects\/[^\/]+\/regions\/([^\/]+)\/subnetworks\/(.+)$/)
|
352
|
+
subnet['az'] = Regexp.last_match[1]
|
353
|
+
subnet['name'] = Regexp.last_match[2]
|
354
|
+
subnet["cloud_id"] = subnet['name']
|
355
|
+
subnet["ip_block"] = desc.ip_cidr_range
|
356
|
+
subnet["url"] = desc.subnetwork
|
357
|
+
subnet['mu_name'] = @mu_name+"-"+subnet['name']
|
358
|
+
if !ext_ids.include?(subnet["cloud_id"])
|
359
|
+
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
|
360
|
+
end
|
361
|
+
}
|
362
|
+
end
|
363
|
+
}
|
364
|
+
end
|
323
365
|
|
324
|
-
if !ext_ids.include?(subnet["cloud_id"])
|
325
|
-
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet)
|
326
|
-
end
|
327
|
-
}
|
328
|
-
|
329
|
-
# Of course we might be loading up a dummy subnet object from a
|
330
|
-
# foreign or non-Mu-created VPC and subnet. So make something up.
|
331
|
-
elsif !found.nil?
|
332
|
-
found.each { |desc|
|
333
|
-
subnet = {}
|
334
|
-
subnet["ip_block"] = desc.ip_cidr_range
|
335
|
-
subnet["name"] = subnet["ip_block"].gsub(/[\.\/]/, "_")
|
336
|
-
subnet['mu_name'] = @mu_name+"-"+subnet['name']
|
337
|
-
subnet["cloud_id"] = desc.name
|
338
|
-
subnet['az'] = subnet['region'] = desc.region.gsub(/.*?\//, "")
|
339
|
-
if !ext_ids.include?(desc.name)
|
340
|
-
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet)
|
341
|
-
end
|
342
|
-
}
|
343
|
-
end
|
344
|
-
|
345
|
-
}
|
346
366
|
return @subnets
|
347
|
-
|
348
367
|
end
|
349
368
|
|
350
369
|
# Given some search criteria try locating a NAT Gaateway in this VPC.
|
@@ -363,7 +382,7 @@ module MU
|
|
363
382
|
# @param nat_tag_value [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_key.
|
364
383
|
# @param nat_ip [String]: An IP address associated with the NAT instance.
|
365
384
|
def findBastion(nat_name: nil, nat_cloud_id: nil, nat_tag_key: nil, nat_tag_value: nil, nat_ip: nil)
|
366
|
-
|
385
|
+
|
367
386
|
deploy_id = nil
|
368
387
|
nat_name = nat_name.to_s if !nat_name.nil? and nat_name.class.to_s == "MU::Config::Tail"
|
369
388
|
nat_ip = nat_ip.to_s if !nat_ip.nil? and nat_ip.class.to_s == "MU::Config::Tail"
|
@@ -397,9 +416,8 @@ module MU
|
|
397
416
|
(cloud_desc.private_ip_address == nat_host_ip or cloud_desc.public_ip_address == nat_host_ip)
|
398
417
|
return nat
|
399
418
|
elsif cloud_desc.vpc_id == @cloud_id
|
400
|
-
# XXX Strictly speaking we could have different NATs in
|
401
|
-
# subnets, so this can be wrong in corner cases.
|
402
|
-
# architect something that obnoxiously, I have no idea.
|
419
|
+
# XXX Strictly speaking we could have different NATs in
|
420
|
+
# different subnets, so this can be wrong in corner cases.
|
403
421
|
return nat
|
404
422
|
end
|
405
423
|
}
|
@@ -412,16 +430,15 @@ module MU
|
|
412
430
|
# Check for a subnet in this VPC matching one or more of the specified
|
413
431
|
# criteria, and return it if found.
|
414
432
|
def getSubnet(cloud_id: nil, name: nil, tag_key: nil, tag_value: nil, ip_block: nil)
|
415
|
-
loadSubnets
|
416
433
|
if !cloud_id.nil? and cloud_id.match(/^https:\/\//)
|
417
434
|
cloud_id.gsub!(/.*?\//, "")
|
418
435
|
end
|
419
436
|
MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block})", MU::DEBUG, details: caller[0]
|
420
|
-
|
421
|
-
@subnets.each { |subnet|
|
437
|
+
subnets.each { |subnet|
|
422
438
|
if !cloud_id.nil? and !subnet.cloud_id.nil? and subnet.cloud_id.to_s == cloud_id.to_s
|
423
439
|
return subnet
|
424
|
-
elsif !name.nil? and !subnet.name.nil? and
|
440
|
+
elsif !name.nil? and !subnet.name.nil? and
|
441
|
+
subnet.name.downcase.to_s == name.downcase.to_s
|
425
442
|
return subnet
|
426
443
|
end
|
427
444
|
}
|
@@ -507,24 +524,157 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
507
524
|
# @return [void]
|
508
525
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
509
526
|
flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
|
527
|
+
return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
|
510
528
|
|
511
529
|
purge_subnets(noop, project: flags['project'], credentials: credentials)
|
512
530
|
["route", "network"].each { |type|
|
513
531
|
# XXX tagged routes aren't showing up in list, and the networks that own them
|
514
532
|
# fail to delete silently
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
533
|
+
retries = 0
|
534
|
+
|
535
|
+
begin
|
536
|
+
MU::Cloud::Google.compute(credentials: credentials).delete(
|
537
|
+
type,
|
538
|
+
flags["project"],
|
539
|
+
nil,
|
540
|
+
noop
|
541
|
+
)
|
542
|
+
rescue MU::MuError, ::Google::Apis::ClientError => e
|
543
|
+
if retries < 5
|
544
|
+
if type == "network"
|
545
|
+
MU.log e.message, MU::WARN
|
546
|
+
if e.message.match(/Failed to delete network (.+)/)
|
547
|
+
network_name = Regexp.last_match[1]
|
548
|
+
fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
|
549
|
+
fwrules.reject! { |name, desc|
|
550
|
+
!desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
|
551
|
+
}
|
552
|
+
fwrules.keys.each { |name|
|
553
|
+
MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE
|
554
|
+
MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name)
|
555
|
+
}
|
556
|
+
end
|
557
|
+
end
|
558
|
+
sleep retries*3
|
559
|
+
retries += 1
|
560
|
+
retry
|
561
|
+
else
|
562
|
+
raise e
|
563
|
+
end
|
564
|
+
end
|
565
|
+
}
|
566
|
+
|
567
|
+
end
|
568
|
+
|
569
|
+
# Reverse-map our cloud description into a runnable config hash.
|
570
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
571
|
+
# calculate our own accordingly based on what's live in the cloud.
|
572
|
+
# XXX add flag to return the diff between @config and live cloud
|
573
|
+
def toKitten(rootparent: nil, billing: nil, habitats: nil)
|
574
|
+
return nil if cloud_desc.name == "default" # parent project builds these
|
575
|
+
bok = {
|
576
|
+
"cloud" => "Google",
|
577
|
+
"project" => @config['project'],
|
578
|
+
"credentials" => @config['credentials']
|
521
579
|
}
|
580
|
+
MU::Cloud::Google.listRegions.size
|
581
|
+
|
582
|
+
diff = {}
|
583
|
+
schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
|
584
|
+
return [nil, nil] if !valid
|
585
|
+
# pp schema
|
586
|
+
# MU.log "++++++++++++++++++++++++++++++++"
|
587
|
+
|
588
|
+
bok['name'] = cloud_desc.name.dup
|
589
|
+
bok['cloud_id'] = cloud_desc.name.dup
|
590
|
+
bok['create_standard_subnets'] = false
|
591
|
+
|
592
|
+
if @subnets and @subnets.size > 0
|
593
|
+
bok['subnets'] = []
|
594
|
+
regions_seen = []
|
595
|
+
names_seen = []
|
596
|
+
@subnets.map { |x| x.cloud_desc }.each { |s|
|
597
|
+
subnet_name = s.name.dup
|
598
|
+
names_seen << s.name.dup
|
599
|
+
regions_seen << s.region
|
600
|
+
bok['subnets'] << {
|
601
|
+
"name" => subnet_name,
|
602
|
+
"ip_block" => s.ip_cidr_range
|
603
|
+
}
|
604
|
+
}
|
605
|
+
|
606
|
+
# If all of the subnets are named 'default' and there's one per
|
607
|
+
# region, we're using GCP-generated subnets instead of explicitly
|
608
|
+
# declared ones.
|
609
|
+
if names_seen.uniq.size == 1 and names_seen.first == "default" and
|
610
|
+
regions_seen.uniq.size == regions_seen.size and
|
611
|
+
regions_seen.size >= (MU::Cloud::Google.listRegions.size * 0.8)
|
612
|
+
bok.delete("subnets")
|
613
|
+
bok['auto_create_subnetworks'] = true
|
614
|
+
end
|
615
|
+
end
|
616
|
+
|
617
|
+
peer_names = []
|
618
|
+
if cloud_desc.peerings and cloud_desc.peerings.size > 0
|
619
|
+
bok['peers'] = []
|
620
|
+
cloud_desc.peerings.each { |peer|
|
621
|
+
peer.network.match(/projects\/([^\/]+?)\/[^\/]+?\/networks\/([^\/]+)$/)
|
622
|
+
vpc_project = Regexp.last_match[1]
|
623
|
+
vpc_name = Regexp.last_match[2]
|
624
|
+
vpc_id = vpc_name.dup
|
625
|
+
# Make sure the peer is something we have permission to look at
|
626
|
+
peer_descs = MU::Cloud::Google::VPC.find(cloud_id: vpc_id, project: vpc_project)
|
627
|
+
if peer_descs.nil? or peer_descs.empty?
|
628
|
+
MU.log "VPC #{@cloud_id} peer #{vpc_id} #{vpc_project} is not accessible, will remove from peer list", MU::WARN
|
629
|
+
next
|
630
|
+
end
|
631
|
+
# XXX need to decide which of these parameters to use based on whether the peer is also in the mix of things being harvested, which is above this method's pay grade
|
632
|
+
bok['peers'] << { "vpc" => MU::Config::Ref.get(
|
633
|
+
id: vpc_id,
|
634
|
+
name: vpc_name,
|
635
|
+
cloud: "Google",
|
636
|
+
habitat: MU::Config::Ref.get(
|
637
|
+
id: vpc_project,
|
638
|
+
cloud: "Google",
|
639
|
+
credentials: @credentials,
|
640
|
+
type: "habitats"
|
641
|
+
),
|
642
|
+
credentials: @config['credentials'],
|
643
|
+
type: "vpcs"
|
644
|
+
) }
|
645
|
+
}
|
646
|
+
end
|
647
|
+
|
648
|
+
# XXX need to grok VPN tunnels, priorities, and maybe preserve descriptions; make sure we know where next_hop_gateway and next_hop_ip come from
|
649
|
+
if @routes
|
650
|
+
routes = []
|
651
|
+
@routes.each { |r|
|
652
|
+
next if r.next_hop_peering # these are auto-created
|
653
|
+
route = {
|
654
|
+
"destination_network" => r.dest_range
|
655
|
+
}
|
656
|
+
if r.next_hop_instance
|
657
|
+
route["nat_host_id"] = r.next_hop_instance
|
658
|
+
end
|
659
|
+
}
|
660
|
+
if routes.size > 0
|
661
|
+
bok['route_tables'] = [
|
662
|
+
{
|
663
|
+
"name" => "default",
|
664
|
+
"routes" => routes
|
665
|
+
}
|
666
|
+
]
|
667
|
+
end
|
668
|
+
end
|
669
|
+
|
670
|
+
# XXX validate that we've at least touched every required attribute (maybe upstream?)
|
671
|
+
bok
|
522
672
|
end
|
523
673
|
|
524
674
|
# Cloud-specific configuration properties.
|
525
675
|
# @param config [MU::Config]: The calling MU::Config object
|
526
676
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
527
|
-
def self.schema(config)
|
677
|
+
def self.schema(config = nil)
|
528
678
|
toplevel_required = []
|
529
679
|
schema = {
|
530
680
|
"regions" => {
|
@@ -533,7 +683,12 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
533
683
|
},
|
534
684
|
"project" => {
|
535
685
|
"type" => "string",
|
536
|
-
"description" => "The project into which to deploy resources"
|
686
|
+
"description" => "The project into which to deploy resources. This is shorthand for a +habitat+ key with a +name+ or +id+ set. The config parser will attempt to correctly resolve this."
|
687
|
+
},
|
688
|
+
"auto_create_subnetworks" => {
|
689
|
+
"type" => "boolean",
|
690
|
+
"default" => false,
|
691
|
+
"description" => "Sets the +auto_create_subnetworks+ flag, which causes Google to generate a set of generic subnets, one per region. This effectively overrides Mu's +create_standard_subnets+ and any explicitly defined +subnets+."
|
537
692
|
}
|
538
693
|
}
|
539
694
|
[toplevel_required, schema]
|
@@ -547,35 +702,10 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
547
702
|
def self.validateConfig(vpc, configurator)
|
548
703
|
ok = true
|
549
704
|
|
705
|
+
vpc['project'] ||= MU::Cloud::Google.defaultProject(vpc['credentials'])
|
550
706
|
|
551
|
-
if vpc[
|
552
|
-
|
553
|
-
if !vpc['route_tables'] or vpc['route_tables'].empty?
|
554
|
-
vpc['route_tables'] = [
|
555
|
-
{
|
556
|
-
"name" => "internet",
|
557
|
-
"routes" => [ { "destination_network" => "0.0.0.0/0", "gateway" => "#INTERNET" } ]
|
558
|
-
},
|
559
|
-
{
|
560
|
-
"name" => "private",
|
561
|
-
"routes" => [ { "destination_network" => "0.0.0.0/0", "gateway" => "#NAT" } ]
|
562
|
-
}
|
563
|
-
]
|
564
|
-
end
|
565
|
-
else
|
566
|
-
# If create_standard_subnets is off, and no route_tables were
|
567
|
-
# declared at all, let's assume we want purely self-contained
|
568
|
-
# private VPC, and create a dummy route accordingly.
|
569
|
-
vpc['route_tables'] ||= [
|
570
|
-
{
|
571
|
-
"name" => "private",
|
572
|
-
"routes" => [
|
573
|
-
{
|
574
|
-
"destination_network" => "0.0.0.0/0"
|
575
|
-
}
|
576
|
-
]
|
577
|
-
}
|
578
|
-
]
|
707
|
+
if vpc["project"] and !vpc["habitat"]
|
708
|
+
vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
|
579
709
|
end
|
580
710
|
|
581
711
|
# Generate a set of subnets per route, if none are declared
|
@@ -588,21 +718,40 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
588
718
|
|
589
719
|
vpc["subnets"] = []
|
590
720
|
vpc['route_tables'].each { |t|
|
721
|
+
is_public = false
|
722
|
+
t['routes'].each { |r|
|
723
|
+
if !vpc["virtual_name"] and !vpc["create_nat_gateway"] and
|
724
|
+
r["gateway"] == "#NAT"
|
725
|
+
r["gateway"] = "#DENY"
|
726
|
+
end
|
727
|
+
is_public = true if r["gateway"] == "#INTERNET"
|
728
|
+
}
|
591
729
|
count = 0
|
592
730
|
vpc['regions'].each { |r|
|
593
731
|
block = blocks.shift
|
594
|
-
|
732
|
+
subnet = {
|
595
733
|
"availability_zone" => r,
|
596
734
|
"route_table" => t["name"],
|
597
735
|
"ip_block" => block.to_s,
|
598
|
-
"name" => "Subnet"+count.to_s+t["name"].capitalize
|
599
|
-
"map_public_ips" => true
|
736
|
+
"name" => "Subnet"+count.to_s+t["name"].capitalize
|
600
737
|
}
|
738
|
+
if is_public
|
739
|
+
subnet["map_public_ips"] = true
|
740
|
+
subnet["is_public"] = true
|
741
|
+
end
|
742
|
+
vpc["subnets"] << subnet
|
601
743
|
count = count + 1
|
602
744
|
}
|
603
745
|
}
|
604
746
|
end
|
605
747
|
|
748
|
+
vpc['subnets'].each { |s|
|
749
|
+
if !s['availability_zone']
|
750
|
+
s['availability_zone'] = vpc['region']
|
751
|
+
s['availability_zone'] ||= MU::Cloud::Google.myRegion(vpc['credentials'])
|
752
|
+
end
|
753
|
+
}
|
754
|
+
|
606
755
|
# Google VPCs can't have routes that are anything other than global
|
607
756
|
# (they can be tied to individual instances by tags, but w/e). So we
|
608
757
|
# decompose our VPCs into littler VPCs, one for each declared route
|
@@ -617,6 +766,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
617
766
|
vpc['route_tables'].each { |tbl|
|
618
767
|
newvpc = {
|
619
768
|
"name" => vpc['name']+"-"+tbl['name'],
|
769
|
+
"cloud" => "Google",
|
620
770
|
"credentials" => vpc['credentials'],
|
621
771
|
"virtual_name" => vpc['name'],
|
622
772
|
"ip_block" => blocks.shift,
|
@@ -645,6 +795,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
645
795
|
vpc["subnets"].each { |subnet|
|
646
796
|
newvpc["subnets"] << subnet if subnet["route_table"] == tbl["name"]
|
647
797
|
}
|
798
|
+
|
648
799
|
ok = false if !configurator.insertKitten(newvpc, "vpcs", true)
|
649
800
|
}
|
650
801
|
configurator.removeKitten(vpc['name'], "vpcs")
|
@@ -678,29 +829,20 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
678
829
|
# No such thing as a NAT gateway in Google... so make an instance
|
679
830
|
# that'll do the deed.
|
680
831
|
if route['gateway'] == "#NAT"
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
|
687
|
-
"nat" => {
|
688
|
-
"private_net" => vpc["parent_block"].to_s
|
689
|
-
}
|
690
|
-
}
|
691
|
-
route['nat_host_name'] = nat_cfg['name']
|
692
|
-
route['priority'] = 100
|
693
|
-
vpc["dependencies"] << {
|
694
|
-
"type" => "server",
|
695
|
-
"name" => nat_cfg['name'],
|
696
|
-
}
|
832
|
+
# theoretically our upstream validation should have inserted
|
833
|
+
# a NAT/bastion host we can use
|
834
|
+
nat = configurator.haveLitterMate?(vpc['name']+"-natstion", "servers")
|
835
|
+
if vpc['virtual_name']
|
836
|
+
nat ||= configurator.haveLitterMate?(vpc['virtual_name']+"-natstion", "servers")
|
837
|
+
end
|
697
838
|
|
698
|
-
|
699
|
-
"
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
839
|
+
if !nat
|
840
|
+
MU.log "Google VPC #{vpc['name']} declared a #NAT route, but I don't see an upstream NAT host I can use. Do I even have public subnets?", MU::ERR
|
841
|
+
ok = false
|
842
|
+
else
|
843
|
+
route['nat_host_name'] = vpc['name']+"-natstion"
|
844
|
+
route['priority'] = 100
|
845
|
+
end
|
704
846
|
end
|
705
847
|
}
|
706
848
|
end
|
@@ -769,7 +911,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
769
911
|
raise MuError, "Failed to find NAT host for #NAT route in #{@mu_name} (#{route})"
|
770
912
|
end
|
771
913
|
|
772
|
-
routeobj = ::Google::Apis::
|
914
|
+
routeobj = ::Google::Apis::ComputeV1::Route.new(
|
773
915
|
name: routename,
|
774
916
|
next_hop_instance: nat_instance.cloud_desc.self_link,
|
775
917
|
dest_range: route['destination_network'],
|
@@ -794,7 +936,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
794
936
|
}
|
795
937
|
end
|
796
938
|
elsif route['gateway'] == "#INTERNET"
|
797
|
-
routeobj = ::Google::Apis::
|
939
|
+
routeobj = ::Google::Apis::ComputeV1::Route.new(
|
798
940
|
name: routename,
|
799
941
|
next_hop_gateway: "global/gateways/default-internet-gateway",
|
800
942
|
dest_range: route['destination_network'],
|
@@ -863,12 +1005,16 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
863
1005
|
regions.each { |r|
|
864
1006
|
regionthreads << Thread.new {
|
865
1007
|
MU.dupGlobals(parent_thread_id)
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
1008
|
+
begin
|
1009
|
+
MU::Cloud::Google.compute(credentials: credentials).delete(
|
1010
|
+
"subnetwork",
|
1011
|
+
project,
|
1012
|
+
r,
|
1013
|
+
noop
|
1014
|
+
)
|
1015
|
+
rescue MU::Cloud::MuDefunctHabitat => e
|
1016
|
+
Thread.exit
|
1017
|
+
end
|
872
1018
|
}
|
873
1019
|
}
|
874
1020
|
regionthreads.each do |t|
|
@@ -888,12 +1034,12 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
888
1034
|
attr_reader :ip_block
|
889
1035
|
attr_reader :mu_name
|
890
1036
|
attr_reader :name
|
1037
|
+
attr_reader :cloud_desc_cache
|
891
1038
|
attr_reader :az
|
892
1039
|
|
893
|
-
|
894
1040
|
# @param parent [MU::Cloud::Google::VPC]: The parent VPC of this subnet.
|
895
1041
|
# @param config [Hash<String>]:
|
896
|
-
def initialize(parent, config)
|
1042
|
+
def initialize(parent, config, precache_description: true)
|
897
1043
|
@parent = parent
|
898
1044
|
@config = MU::Config.manxify(config)
|
899
1045
|
@cloud_id = config['cloud_id']
|
@@ -903,20 +1049,32 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
903
1049
|
@deploydata = config # This is a dummy for the sake of describe()
|
904
1050
|
@az = config['az']
|
905
1051
|
@ip_block = config['ip_block']
|
1052
|
+
@cloud_desc_cache = nil
|
1053
|
+
cloud_desc if precache_description
|
906
1054
|
end
|
907
1055
|
|
908
1056
|
# Return the cloud identifier for the default route of this subnet.
|
909
1057
|
def defaultRoute
|
910
1058
|
end
|
911
1059
|
|
1060
|
+
# Describe this VPC Subnet
|
1061
|
+
# @return [Hash]
|
1062
|
+
def notify
|
1063
|
+
MU.structToHash(cloud_desc)
|
1064
|
+
end
|
1065
|
+
|
1066
|
+
# Describe this VPC Subnet from the cloud platform's perspective
|
1067
|
+
# @return [Google::Apis::Core::Hashable]
|
1068
|
+
def cloud_desc
|
1069
|
+
@cloud_desc_cache ||= MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @config['az'], @config['cloud_id'])
|
1070
|
+
@cloud_desc_cache
|
1071
|
+
end
|
1072
|
+
|
912
1073
|
# Is this subnet privately-routable only, or public?
|
913
1074
|
# @return [Boolean]
|
914
1075
|
def private?
|
915
|
-
|
916
|
-
|
917
|
-
filter: "network eq #{@parent.url}"
|
918
|
-
).items
|
919
|
-
routes.map { |r|
|
1076
|
+
@parent.cloud_desc
|
1077
|
+
@parent.routes.map { |r|
|
920
1078
|
if r.dest_range == "0.0.0.0/0" and !r.next_hop_gateway.nil? and
|
921
1079
|
(r.tags.nil? or r.tags.size == 0) and
|
922
1080
|
r.next_hop_gateway.match(/\/global\/gateways\/default-internet-gateway/)
|