cloud-mu 2.1.0beta → 3.0.0beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Berksfile +4 -5
- data/Berksfile.lock +179 -0
- data/README.md +1 -6
- data/ansible/roles/geerlingguy.firewall/templates/firewall.bash.j2 +0 -0
- data/ansible/roles/mu-installer/README.md +33 -0
- data/ansible/roles/mu-installer/defaults/main.yml +2 -0
- data/ansible/roles/mu-installer/handlers/main.yml +2 -0
- data/ansible/roles/mu-installer/meta/main.yml +60 -0
- data/ansible/roles/mu-installer/tasks/main.yml +13 -0
- data/ansible/roles/mu-installer/tests/inventory +2 -0
- data/ansible/roles/mu-installer/tests/test.yml +5 -0
- data/ansible/roles/mu-installer/vars/main.yml +2 -0
- data/bin/mu-adopt +125 -0
- data/bin/mu-aws-setup +4 -4
- data/bin/mu-azure-setup +265 -0
- data/bin/mu-azure-tests +43 -0
- data/bin/mu-cleanup +20 -8
- data/bin/mu-configure +224 -98
- data/bin/mu-deploy +8 -3
- data/bin/mu-gcp-setup +16 -8
- data/bin/mu-gen-docs +92 -8
- data/bin/mu-load-config.rb +52 -12
- data/bin/mu-momma-cat +36 -0
- data/bin/mu-node-manage +34 -27
- data/bin/mu-self-update +2 -2
- data/bin/mu-ssh +12 -8
- data/bin/mu-upload-chef-artifacts +11 -4
- data/bin/mu-user-manage +3 -0
- data/cloud-mu.gemspec +8 -11
- data/cookbooks/firewall/libraries/helpers_iptables.rb +2 -2
- data/cookbooks/firewall/metadata.json +1 -1
- data/cookbooks/firewall/recipes/default.rb +5 -9
- data/cookbooks/mu-firewall/attributes/default.rb +2 -0
- data/cookbooks/mu-firewall/metadata.rb +1 -1
- data/cookbooks/mu-glusterfs/templates/default/mu-gluster-client.erb +0 -0
- data/cookbooks/mu-master/Berksfile +2 -2
- data/cookbooks/mu-master/files/default/check_mem.pl +0 -0
- data/cookbooks/mu-master/files/default/cloudamatic.png +0 -0
- data/cookbooks/mu-master/metadata.rb +5 -4
- data/cookbooks/mu-master/recipes/389ds.rb +1 -1
- data/cookbooks/mu-master/recipes/basepackages.rb +30 -10
- data/cookbooks/mu-master/recipes/default.rb +59 -7
- data/cookbooks/mu-master/recipes/firewall-holes.rb +1 -1
- data/cookbooks/mu-master/recipes/init.rb +65 -47
- data/cookbooks/mu-master/recipes/{eks-kubectl.rb → kubectl.rb} +4 -10
- data/cookbooks/mu-master/recipes/sssd.rb +2 -1
- data/cookbooks/mu-master/recipes/update_nagios_only.rb +6 -6
- data/cookbooks/mu-master/templates/default/web_app.conf.erb +2 -2
- data/cookbooks/mu-master/templates/mods/ldap.conf.erb +4 -0
- data/cookbooks/mu-php54/Berksfile +1 -2
- data/cookbooks/mu-php54/metadata.rb +4 -5
- data/cookbooks/mu-php54/recipes/default.rb +1 -1
- data/cookbooks/mu-splunk/templates/default/splunk-init.erb +0 -0
- data/cookbooks/mu-tools/Berksfile +3 -2
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +33 -0
- data/cookbooks/mu-tools/libraries/helper.rb +20 -8
- data/cookbooks/mu-tools/metadata.rb +5 -2
- data/cookbooks/mu-tools/recipes/apply_security.rb +2 -3
- data/cookbooks/mu-tools/recipes/eks.rb +1 -1
- data/cookbooks/mu-tools/recipes/gcloud.rb +5 -30
- data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
- data/cookbooks/mu-tools/recipes/rsyslog.rb +1 -0
- data/cookbooks/mu-tools/recipes/selinux.rb +19 -0
- data/cookbooks/mu-tools/recipes/split_var_partitions.rb +0 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +256 -122
- data/cookbooks/mu-tools/resources/disk.rb +3 -1
- data/cookbooks/mu-tools/templates/amazon/sshd_config.erb +1 -1
- data/cookbooks/mu-tools/templates/default/etc_hosts.erb +1 -1
- data/cookbooks/mu-tools/templates/default/{kubeconfig.erb → kubeconfig-eks.erb} +0 -0
- data/cookbooks/mu-tools/templates/default/kubeconfig-gke.erb +27 -0
- data/cookbooks/mu-tools/templates/windows-10/sshd_config.erb +137 -0
- data/cookbooks/mu-utility/recipes/nat.rb +4 -0
- data/extras/alpha.png +0 -0
- data/extras/beta.png +0 -0
- data/extras/clean-stock-amis +2 -2
- data/extras/generate-stock-images +131 -0
- data/extras/git-fix-permissions-hook +0 -0
- data/extras/image-generators/AWS/centos6.yaml +17 -0
- data/extras/image-generators/{aws → AWS}/centos7-govcloud.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/centos7.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/rhel7.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/win2k12.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/win2k16.yaml +0 -0
- data/extras/image-generators/{aws → AWS}/windows.yaml +0 -0
- data/extras/image-generators/{gcp → Google}/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +18 -0
- data/extras/python_rpm/build.sh +0 -0
- data/extras/release.png +0 -0
- data/extras/ruby_rpm/build.sh +0 -0
- data/extras/ruby_rpm/muby.spec +1 -1
- data/install/README.md +43 -5
- data/install/deprecated-bash-library.sh +0 -0
- data/install/installer +1 -1
- data/install/jenkinskeys.rb +0 -0
- data/install/mu-master.yaml +55 -0
- data/modules/mommacat.ru +41 -7
- data/modules/mu.rb +444 -149
- data/modules/mu/adoption.rb +500 -0
- data/modules/mu/cleanup.rb +235 -158
- data/modules/mu/cloud.rb +675 -138
- data/modules/mu/clouds/aws.rb +156 -24
- data/modules/mu/clouds/aws/alarm.rb +4 -14
- data/modules/mu/clouds/aws/bucket.rb +60 -18
- data/modules/mu/clouds/aws/cache_cluster.rb +8 -20
- data/modules/mu/clouds/aws/collection.rb +12 -22
- data/modules/mu/clouds/aws/container_cluster.rb +209 -118
- data/modules/mu/clouds/aws/database.rb +120 -45
- data/modules/mu/clouds/aws/dnszone.rb +7 -18
- data/modules/mu/clouds/aws/endpoint.rb +5 -15
- data/modules/mu/clouds/aws/firewall_rule.rb +144 -72
- data/modules/mu/clouds/aws/folder.rb +4 -11
- data/modules/mu/clouds/aws/function.rb +6 -16
- data/modules/mu/clouds/aws/group.rb +4 -12
- data/modules/mu/clouds/aws/habitat.rb +11 -13
- data/modules/mu/clouds/aws/loadbalancer.rb +40 -28
- data/modules/mu/clouds/aws/log.rb +5 -13
- data/modules/mu/clouds/aws/msg_queue.rb +9 -24
- data/modules/mu/clouds/aws/nosqldb.rb +4 -12
- data/modules/mu/clouds/aws/notifier.rb +6 -13
- data/modules/mu/clouds/aws/role.rb +69 -40
- data/modules/mu/clouds/aws/search_domain.rb +17 -20
- data/modules/mu/clouds/aws/server.rb +184 -94
- data/modules/mu/clouds/aws/server_pool.rb +33 -38
- data/modules/mu/clouds/aws/storage_pool.rb +5 -12
- data/modules/mu/clouds/aws/user.rb +59 -33
- data/modules/mu/clouds/aws/userdata/linux.erb +18 -30
- data/modules/mu/clouds/aws/userdata/windows.erb +9 -9
- data/modules/mu/clouds/aws/vpc.rb +214 -145
- data/modules/mu/clouds/azure.rb +978 -44
- data/modules/mu/clouds/azure/container_cluster.rb +413 -0
- data/modules/mu/clouds/azure/firewall_rule.rb +500 -0
- data/modules/mu/clouds/azure/habitat.rb +167 -0
- data/modules/mu/clouds/azure/loadbalancer.rb +205 -0
- data/modules/mu/clouds/azure/role.rb +211 -0
- data/modules/mu/clouds/azure/server.rb +810 -0
- data/modules/mu/clouds/azure/user.rb +257 -0
- data/modules/mu/clouds/azure/userdata/README.md +4 -0
- data/modules/mu/clouds/azure/userdata/linux.erb +137 -0
- data/modules/mu/clouds/azure/userdata/windows.erb +275 -0
- data/modules/mu/clouds/azure/vpc.rb +782 -0
- data/modules/mu/clouds/cloudformation.rb +12 -9
- data/modules/mu/clouds/cloudformation/firewall_rule.rb +5 -13
- data/modules/mu/clouds/cloudformation/server.rb +10 -1
- data/modules/mu/clouds/cloudformation/server_pool.rb +1 -0
- data/modules/mu/clouds/cloudformation/vpc.rb +0 -2
- data/modules/mu/clouds/google.rb +554 -117
- data/modules/mu/clouds/google/bucket.rb +173 -32
- data/modules/mu/clouds/google/container_cluster.rb +1112 -157
- data/modules/mu/clouds/google/database.rb +24 -47
- data/modules/mu/clouds/google/firewall_rule.rb +344 -89
- data/modules/mu/clouds/google/folder.rb +156 -79
- data/modules/mu/clouds/google/group.rb +272 -82
- data/modules/mu/clouds/google/habitat.rb +177 -52
- data/modules/mu/clouds/google/loadbalancer.rb +9 -34
- data/modules/mu/clouds/google/role.rb +1211 -0
- data/modules/mu/clouds/google/server.rb +491 -227
- data/modules/mu/clouds/google/server_pool.rb +233 -48
- data/modules/mu/clouds/google/user.rb +479 -125
- data/modules/mu/clouds/google/userdata/linux.erb +3 -3
- data/modules/mu/clouds/google/userdata/windows.erb +9 -9
- data/modules/mu/clouds/google/vpc.rb +381 -223
- data/modules/mu/config.rb +689 -214
- data/modules/mu/config/bucket.rb +1 -1
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/cache_cluster.yml +0 -4
- data/modules/mu/config/container_cluster.rb +18 -9
- data/modules/mu/config/database.rb +6 -23
- data/modules/mu/config/firewall_rule.rb +9 -15
- data/modules/mu/config/folder.rb +22 -21
- data/modules/mu/config/habitat.rb +22 -21
- data/modules/mu/config/loadbalancer.rb +2 -2
- data/modules/mu/config/role.rb +9 -40
- data/modules/mu/config/server.rb +26 -5
- data/modules/mu/config/server_pool.rb +1 -1
- data/modules/mu/config/storage_pool.rb +2 -2
- data/modules/mu/config/user.rb +4 -0
- data/modules/mu/config/vpc.rb +350 -110
- data/modules/mu/defaults/{amazon_images.yaml → AWS.yaml} +37 -39
- data/modules/mu/defaults/Azure.yaml +17 -0
- data/modules/mu/defaults/Google.yaml +24 -0
- data/modules/mu/defaults/README.md +1 -1
- data/modules/mu/deploy.rb +168 -125
- data/modules/mu/groomer.rb +2 -1
- data/modules/mu/groomers/ansible.rb +104 -32
- data/modules/mu/groomers/chef.rb +96 -44
- data/modules/mu/kittens.rb +20602 -0
- data/modules/mu/logger.rb +38 -11
- data/modules/mu/master.rb +90 -8
- data/modules/mu/master/chef.rb +2 -3
- data/modules/mu/master/ldap.rb +0 -1
- data/modules/mu/master/ssl.rb +250 -0
- data/modules/mu/mommacat.rb +917 -513
- data/modules/scratchpad.erb +1 -1
- data/modules/tests/super_complex_bok.yml +0 -0
- data/modules/tests/super_simple_bok.yml +0 -0
- data/roles/mu-master.json +2 -1
- data/spec/azure_creds +5 -0
- data/spec/mu.yaml +56 -0
- data/spec/mu/clouds/azure_spec.rb +164 -27
- data/spec/spec_helper.rb +5 -0
- data/test/clean_up.py +0 -0
- data/test/exec_inspec.py +0 -0
- data/test/exec_mu_install.py +0 -0
- data/test/exec_retry.py +0 -0
- data/test/smoke_test.rb +0 -0
- metadata +90 -118
- data/cookbooks/mu-jenkins/Berksfile +0 -14
- data/cookbooks/mu-jenkins/CHANGELOG.md +0 -13
- data/cookbooks/mu-jenkins/LICENSE +0 -37
- data/cookbooks/mu-jenkins/README.md +0 -105
- data/cookbooks/mu-jenkins/attributes/default.rb +0 -42
- data/cookbooks/mu-jenkins/files/default/cleanup_deploy_config.xml +0 -73
- data/cookbooks/mu-jenkins/files/default/deploy_config.xml +0 -44
- data/cookbooks/mu-jenkins/metadata.rb +0 -21
- data/cookbooks/mu-jenkins/recipes/default.rb +0 -195
- data/cookbooks/mu-jenkins/recipes/node-ssh-config.rb +0 -54
- data/cookbooks/mu-jenkins/recipes/public_key.rb +0 -24
- data/cookbooks/mu-jenkins/templates/default/example_job.config.xml.erb +0 -24
- data/cookbooks/mu-jenkins/templates/default/org.jvnet.hudson.plugins.SSHBuildWrapper.xml.erb +0 -14
- data/cookbooks/mu-jenkins/templates/default/ssh_config.erb +0 -6
- data/cookbooks/nagios/Berksfile +0 -11
- data/cookbooks/nagios/CHANGELOG.md +0 -589
- data/cookbooks/nagios/CONTRIBUTING.md +0 -11
- data/cookbooks/nagios/LICENSE +0 -37
- data/cookbooks/nagios/README.md +0 -328
- data/cookbooks/nagios/TESTING.md +0 -2
- data/cookbooks/nagios/attributes/config.rb +0 -171
- data/cookbooks/nagios/attributes/default.rb +0 -228
- data/cookbooks/nagios/chefignore +0 -102
- data/cookbooks/nagios/definitions/command.rb +0 -33
- data/cookbooks/nagios/definitions/contact.rb +0 -33
- data/cookbooks/nagios/definitions/contactgroup.rb +0 -33
- data/cookbooks/nagios/definitions/host.rb +0 -33
- data/cookbooks/nagios/definitions/hostdependency.rb +0 -33
- data/cookbooks/nagios/definitions/hostescalation.rb +0 -34
- data/cookbooks/nagios/definitions/hostgroup.rb +0 -33
- data/cookbooks/nagios/definitions/nagios_conf.rb +0 -38
- data/cookbooks/nagios/definitions/resource.rb +0 -33
- data/cookbooks/nagios/definitions/service.rb +0 -33
- data/cookbooks/nagios/definitions/servicedependency.rb +0 -33
- data/cookbooks/nagios/definitions/serviceescalation.rb +0 -34
- data/cookbooks/nagios/definitions/servicegroup.rb +0 -33
- data/cookbooks/nagios/definitions/timeperiod.rb +0 -33
- data/cookbooks/nagios/libraries/base.rb +0 -314
- data/cookbooks/nagios/libraries/command.rb +0 -91
- data/cookbooks/nagios/libraries/contact.rb +0 -230
- data/cookbooks/nagios/libraries/contactgroup.rb +0 -112
- data/cookbooks/nagios/libraries/custom_option.rb +0 -36
- data/cookbooks/nagios/libraries/data_bag_helper.rb +0 -23
- data/cookbooks/nagios/libraries/default.rb +0 -90
- data/cookbooks/nagios/libraries/host.rb +0 -412
- data/cookbooks/nagios/libraries/hostdependency.rb +0 -181
- data/cookbooks/nagios/libraries/hostescalation.rb +0 -173
- data/cookbooks/nagios/libraries/hostgroup.rb +0 -119
- data/cookbooks/nagios/libraries/nagios.rb +0 -282
- data/cookbooks/nagios/libraries/resource.rb +0 -59
- data/cookbooks/nagios/libraries/service.rb +0 -455
- data/cookbooks/nagios/libraries/servicedependency.rb +0 -215
- data/cookbooks/nagios/libraries/serviceescalation.rb +0 -195
- data/cookbooks/nagios/libraries/servicegroup.rb +0 -144
- data/cookbooks/nagios/libraries/timeperiod.rb +0 -160
- data/cookbooks/nagios/libraries/users_helper.rb +0 -54
- data/cookbooks/nagios/metadata.rb +0 -25
- data/cookbooks/nagios/recipes/_load_databag_config.rb +0 -153
- data/cookbooks/nagios/recipes/_load_default_config.rb +0 -241
- data/cookbooks/nagios/recipes/apache.rb +0 -48
- data/cookbooks/nagios/recipes/default.rb +0 -204
- data/cookbooks/nagios/recipes/nginx.rb +0 -82
- data/cookbooks/nagios/recipes/pagerduty.rb +0 -143
- data/cookbooks/nagios/recipes/server_package.rb +0 -40
- data/cookbooks/nagios/recipes/server_source.rb +0 -164
- data/cookbooks/nagios/templates/default/apache2.conf.erb +0 -96
- data/cookbooks/nagios/templates/default/cgi.cfg.erb +0 -266
- data/cookbooks/nagios/templates/default/commands.cfg.erb +0 -13
- data/cookbooks/nagios/templates/default/contacts.cfg.erb +0 -37
- data/cookbooks/nagios/templates/default/hostgroups.cfg.erb +0 -25
- data/cookbooks/nagios/templates/default/hosts.cfg.erb +0 -15
- data/cookbooks/nagios/templates/default/htpasswd.users.erb +0 -6
- data/cookbooks/nagios/templates/default/nagios.cfg.erb +0 -22
- data/cookbooks/nagios/templates/default/nginx.conf.erb +0 -62
- data/cookbooks/nagios/templates/default/pagerduty.cgi.erb +0 -185
- data/cookbooks/nagios/templates/default/resource.cfg.erb +0 -27
- data/cookbooks/nagios/templates/default/servicedependencies.cfg.erb +0 -15
- data/cookbooks/nagios/templates/default/servicegroups.cfg.erb +0 -14
- data/cookbooks/nagios/templates/default/services.cfg.erb +0 -14
- data/cookbooks/nagios/templates/default/templates.cfg.erb +0 -31
- data/cookbooks/nagios/templates/default/timeperiods.cfg.erb +0 -13
- data/extras/image-generators/aws/centos6.yaml +0 -18
- data/modules/mu/defaults/google_images.yaml +0 -16
- data/roles/mu-master-jenkins.json +0 -24
@@ -0,0 +1,500 @@
|
|
1
|
+
# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
|
2
|
+
#
|
3
|
+
# Licensed under the BSD-3 license (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License in the root of the project or at
|
6
|
+
#
|
7
|
+
# http://egt-labs.com/mu/LICENSE.html
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module MU
|
16
|
+
|
17
|
+
# Scrape cloud providers for existing resources, and reverse-engineer them
|
18
|
+
# into runnable {MU::Config} descriptors and/or {MU::MommaCat} deploy objects.
|
19
|
+
class Adoption
|
20
|
+
|
21
|
+
attr_reader :found
|
22
|
+
|
23
|
+
# Error class for objects which fail to fully resolve (e.g. references to
|
24
|
+
# other objects which are not found)
|
25
|
+
class Incomplete < MU::MuNonFatal; end
|
26
|
+
|
27
|
+
# Presets methods we use to clump discovered resources into discrete deploys
|
28
|
+
GROUPMODES = {
|
29
|
+
:logical => "Group resources in logical layers (folders and habitats together, users/roles/groups together, network resources together, etc)",
|
30
|
+
:omnibus => "Jam everything into one monolothic configuration"
|
31
|
+
}
|
32
|
+
|
33
|
+
def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: true, diff: false, habitats: [])
|
34
|
+
@scraped = {}
|
35
|
+
@clouds = clouds
|
36
|
+
@types = types
|
37
|
+
@parent = parent
|
38
|
+
@boks = {}
|
39
|
+
@billing = billing
|
40
|
+
@reference_map = {}
|
41
|
+
@sources = sources
|
42
|
+
@target_creds = credentials
|
43
|
+
@group_by = group_by
|
44
|
+
@savedeploys = savedeploys
|
45
|
+
@diff = diff
|
46
|
+
@habitats = habitats
|
47
|
+
@habitats ||= []
|
48
|
+
end
|
49
|
+
|
50
|
+
# Walk cloud providers with available credentials to discover resources
|
51
|
+
def scrapeClouds()
|
52
|
+
@default_parent = nil
|
53
|
+
|
54
|
+
@clouds.each { |cloud|
|
55
|
+
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
|
56
|
+
next if cloudclass.listCredentials.nil?
|
57
|
+
|
58
|
+
if cloud == "Google" and !@parent and @target_creds
|
59
|
+
dest_org = MU::Cloud::Google.getOrg(@target_creds)
|
60
|
+
if dest_org
|
61
|
+
@default_parent = dest_org.name
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
cloudclass.listCredentials.each { |credset|
|
66
|
+
next if @sources and !@sources.include?(credset)
|
67
|
+
|
68
|
+
if @parent
|
69
|
+
# TODO handle different inputs (cloud_id, etc)
|
70
|
+
# TODO do something about vague matches
|
71
|
+
found = MU::MommaCat.findStray(
|
72
|
+
cloud,
|
73
|
+
"folders",
|
74
|
+
flags: { "display_name" => @parent },
|
75
|
+
credentials: credset,
|
76
|
+
allow_multi: false,
|
77
|
+
dummy_ok: true,
|
78
|
+
debug: false
|
79
|
+
)
|
80
|
+
if found and found.size == 1
|
81
|
+
@default_parent = found.first
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
@types.each { |type|
|
86
|
+
begin
|
87
|
+
resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
|
88
|
+
rescue ::MU::Cloud::MuCloudResourceNotImplemented
|
89
|
+
next
|
90
|
+
end
|
91
|
+
if !resclass.instance_methods.include?(:toKitten)
|
92
|
+
MU.log "Skipping MU::Cloud::#{cloud}::#{type} (resource has not implemented #toKitten)", MU::WARN
|
93
|
+
next
|
94
|
+
end
|
95
|
+
MU.log "Scraping #{cloud}/#{credset} for #{resclass.cfg_plural}"
|
96
|
+
|
97
|
+
found = MU::MommaCat.findStray(
|
98
|
+
cloud,
|
99
|
+
type,
|
100
|
+
credentials: credset,
|
101
|
+
allow_multi: true,
|
102
|
+
habitats: @habitats.dup,
|
103
|
+
dummy_ok: true,
|
104
|
+
debug: false,
|
105
|
+
flags: { "skip_provider_owned" => true }
|
106
|
+
)
|
107
|
+
|
108
|
+
|
109
|
+
if found and found.size > 0
|
110
|
+
MU.log "Found #{found.size.to_s} raw #{resclass.cfg_plural} in #{cloud}"
|
111
|
+
@scraped[type] ||= {}
|
112
|
+
found.each { |obj|
|
113
|
+
# XXX apply any filters (e.g. MU-ID tags)
|
114
|
+
@scraped[type][obj.cloud_id] = obj
|
115
|
+
}
|
116
|
+
end
|
117
|
+
|
118
|
+
}
|
119
|
+
}
|
120
|
+
}
|
121
|
+
|
122
|
+
if @parent and !@default_parent
|
123
|
+
MU.log "Failed to locate a folder that resembles #{@parent}", MU::ERR
|
124
|
+
end
|
125
|
+
MU.log "Scraping complete"
|
126
|
+
end
|
127
|
+
|
128
|
+
# Generate a {MU::Config} (Basket of Kittens) hash using our discovered
|
129
|
+
# cloud objects.
|
130
|
+
# @return [Hash]
|
131
|
+
def generateBaskets(prefix: "")
|
132
|
+
groupings = {
|
133
|
+
"" => MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
|
134
|
+
}
|
135
|
+
|
136
|
+
# XXX as soon as we come up with a method that isn't about what resource
|
137
|
+
# type you are, this code will stop making sense
|
138
|
+
if @group_by == :logical
|
139
|
+
groupings = {
|
140
|
+
"spaces" => ["folders", "habitats"],
|
141
|
+
"people" => ["users", "groups", "roles"],
|
142
|
+
"network" => ["vpcs", "firewall_rules", "dnszones"],
|
143
|
+
"storage" => ["storage_pools", "buckets"],
|
144
|
+
}
|
145
|
+
# "the movie star/and the rest"
|
146
|
+
groupings["services"] = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] } - groupings.values.flatten
|
147
|
+
elsif @group_by == :omnibus
|
148
|
+
prefix = "mu" if prefix.empty? # so that appnames aren't ever empty
|
149
|
+
end
|
150
|
+
|
151
|
+
groupings.each_pair { |appname, types|
|
152
|
+
bok = { "appname" => prefix+appname }
|
153
|
+
if @target_creds
|
154
|
+
bok["credentials"] = @target_creds
|
155
|
+
end
|
156
|
+
|
157
|
+
count = 0
|
158
|
+
allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
|
159
|
+
next if (types & allowed_types).size == 0
|
160
|
+
origin = {
|
161
|
+
"appname" => bok['appname'],
|
162
|
+
"types" => (types & allowed_types).sort,
|
163
|
+
"habitats" => @habitats.sort,
|
164
|
+
"group_by" => @group_by.to_s
|
165
|
+
}
|
166
|
+
|
167
|
+
deploy = MU::MommaCat.findMatchingDeploy(origin)
|
168
|
+
if @diff and !deploy
|
169
|
+
MU.log "--diff was set but I failed to find a deploy like me to compare to", MU::ERR, details: origin
|
170
|
+
exit 1
|
171
|
+
end
|
172
|
+
|
173
|
+
threads = []
|
174
|
+
@clouds.each { |cloud|
|
175
|
+
@scraped.each_pair { |type, resources|
|
176
|
+
res_class = begin
|
177
|
+
MU::Cloud.loadCloudType(cloud, type)
|
178
|
+
rescue MU::Cloud::MuCloudResourceNotImplemented => e
|
179
|
+
# XXX I don't think this can actually happen
|
180
|
+
next
|
181
|
+
end
|
182
|
+
next if !types.include?(res_class.cfg_plural)
|
183
|
+
|
184
|
+
bok[res_class.cfg_plural] ||= []
|
185
|
+
|
186
|
+
class_semaphore = Mutex.new
|
187
|
+
|
188
|
+
Thread.abort_on_exception = true
|
189
|
+
resources.each_pair { |cloud_id_thr, obj_thr|
|
190
|
+
threads << Thread.new(cloud_id_thr, obj_thr) { |cloud_id, obj|
|
191
|
+
|
192
|
+
kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats)
|
193
|
+
if kitten_cfg
|
194
|
+
print "."
|
195
|
+
kitten_cfg.delete("credentials") if @target_creds
|
196
|
+
class_semaphore.synchronize {
|
197
|
+
bok[res_class.cfg_plural] << kitten_cfg
|
198
|
+
}
|
199
|
+
count += 1
|
200
|
+
end
|
201
|
+
}
|
202
|
+
|
203
|
+
}
|
204
|
+
|
205
|
+
threads.each { |t|
|
206
|
+
t.join
|
207
|
+
}
|
208
|
+
puts ""
|
209
|
+
bok[res_class.cfg_plural].sort! { |a, b|
|
210
|
+
strs = [a, b].map { |x|
|
211
|
+
if x['cloud_id']
|
212
|
+
x['cloud_id']
|
213
|
+
elsif x['parent'] and ['parent'].respond_to?(:id) and kitten_cfg['parent'].id
|
214
|
+
x['name']+x['parent'].id
|
215
|
+
elsif x['project']
|
216
|
+
x['name']+x['project']
|
217
|
+
else
|
218
|
+
x['name']
|
219
|
+
end
|
220
|
+
}
|
221
|
+
strs[0] <=> strs[1]
|
222
|
+
}
|
223
|
+
|
224
|
+
# If we've got duplicate names in here, try to deal with it
|
225
|
+
bok[res_class.cfg_plural].each { |kitten_cfg|
|
226
|
+
bok[res_class.cfg_plural].each { |sibling|
|
227
|
+
next if kitten_cfg == sibling
|
228
|
+
if sibling['name'] == kitten_cfg['name']
|
229
|
+
MU.log "#{res_class.cfg_name} name #{sibling['name']} unavailable, will attempt to rename duplicate object", MU::DEBUG, details: kitten_cfg
|
230
|
+
if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
|
231
|
+
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['parent'].id
|
232
|
+
elsif kitten_cfg['project']
|
233
|
+
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['project']
|
234
|
+
elsif kitten_cfg['cloud_id']
|
235
|
+
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
|
236
|
+
else
|
237
|
+
raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{sibling['name']} and couldn't come up with a good way to differentiate them"
|
238
|
+
end
|
239
|
+
MU.log "De-duplication: Renamed #{res_class.cfg_name} name '#{sibling['name']}' => '#{kitten_cfg['name']}'", MU::NOTICE
|
240
|
+
break
|
241
|
+
end
|
242
|
+
}
|
243
|
+
}
|
244
|
+
}
|
245
|
+
}
|
246
|
+
|
247
|
+
# No matching resources isn't necessarily an error
|
248
|
+
next if count == 0 or bok.nil?
|
249
|
+
|
250
|
+
# Now walk through all of the Refs in these objects, resolve them, and minimize
|
251
|
+
# their config footprint
|
252
|
+
MU.log "Minimizing footprint of #{count.to_s} found resources", MU::DEBUG
|
253
|
+
@boks[bok['appname']] = vacuum(bok, origin: origin, save: @savedeploys)
|
254
|
+
|
255
|
+
if @diff and !deploy
|
256
|
+
MU.log "diff flag set, but no comparable deploy provided for #{bok['appname']}", MU::ERR
|
257
|
+
exit 1
|
258
|
+
end
|
259
|
+
|
260
|
+
if deploy and @diff
|
261
|
+
prevcfg = MU::Config.manxify(vacuum(deploy.original_config, deploy: deploy))
|
262
|
+
if !prevcfg
|
263
|
+
MU.log "#{deploy.deploy_id} didn't have a working original config for me to compare", MU::ERR
|
264
|
+
exit 1
|
265
|
+
end
|
266
|
+
newcfg = MU::Config.manxify(@boks[bok['appname']])
|
267
|
+
|
268
|
+
prevcfg.diff(newcfg)
|
269
|
+
exit
|
270
|
+
end
|
271
|
+
}
|
272
|
+
@boks
|
273
|
+
end
|
274
|
+
|
275
|
+
private
|
276
|
+
|
277
|
+
# Recursively walk through a BoK hash, validate all {MU::Config::Ref}
|
278
|
+
# objects, convert them to hashes, and pare them down to the minimal
|
279
|
+
# representation (remove extraneous attributes that match the parent
|
280
|
+
# object).
|
281
|
+
# Do the same for our main objects: if they all use the same credentials,
|
282
|
+
# for example, remove the explicit +credentials+ attributes and set that
|
283
|
+
# value globally, once.
|
284
|
+
def vacuum(bok, origin: nil, save: false, deploy: nil)
|
285
|
+
deploy ||= generateStubDeploy(bok)
|
286
|
+
|
287
|
+
globals = {
|
288
|
+
'cloud' => {},
|
289
|
+
'credentials' => {},
|
290
|
+
'region' => {},
|
291
|
+
'billing_acct' => {},
|
292
|
+
'us_only' => {},
|
293
|
+
}
|
294
|
+
clouds = {}
|
295
|
+
credentials = {}
|
296
|
+
regions = {}
|
297
|
+
MU::Cloud.resource_types.each_pair { |typename, attrs|
|
298
|
+
if bok[attrs[:cfg_plural]]
|
299
|
+
processed = []
|
300
|
+
bok[attrs[:cfg_plural]].each { |resource|
|
301
|
+
globals.each_pair { |field, counts|
|
302
|
+
if resource[field]
|
303
|
+
counts[resource[field]] ||= 0
|
304
|
+
counts[resource[field]] += 1
|
305
|
+
end
|
306
|
+
}
|
307
|
+
obj = deploy.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
|
308
|
+
begin
|
309
|
+
processed << resolveReferences(resource, deploy, obj)
|
310
|
+
rescue Incomplete
|
311
|
+
end
|
312
|
+
resource.delete("cloud_id")
|
313
|
+
}
|
314
|
+
deploy.original_config[attrs[:cfg_plural]] = processed
|
315
|
+
bok[attrs[:cfg_plural]] = processed
|
316
|
+
end
|
317
|
+
}
|
318
|
+
|
319
|
+
# Pare out global values like +cloud+ or +region+ that appear to be
|
320
|
+
# universal in the deploy we're creating.
|
321
|
+
def scrub_globals(h, field)
|
322
|
+
if h.is_a?(Hash)
|
323
|
+
newhash = {}
|
324
|
+
h.each_pair { |k, v|
|
325
|
+
next if k == field
|
326
|
+
newhash[k] = scrub_globals(v, field)
|
327
|
+
}
|
328
|
+
h = newhash
|
329
|
+
elsif h.is_a?(Array)
|
330
|
+
newarr = []
|
331
|
+
h.each { |v|
|
332
|
+
newarr << scrub_globals(v, field)
|
333
|
+
}
|
334
|
+
h = newarr
|
335
|
+
end
|
336
|
+
|
337
|
+
h
|
338
|
+
end
|
339
|
+
|
340
|
+
globals.each_pair { |field, counts|
|
341
|
+
next if counts.size != 1
|
342
|
+
bok[field] = counts.keys.first
|
343
|
+
MU.log "Setting global default #{field} to #{bok[field]} (#{deploy.deploy_id})", MU::DEBUG
|
344
|
+
MU::Cloud.resource_types.each_pair { |typename, attrs|
|
345
|
+
if bok[attrs[:cfg_plural]]
|
346
|
+
new_resources = []
|
347
|
+
bok[attrs[:cfg_plural]].each { |resource|
|
348
|
+
new_resources << scrub_globals(resource, field)
|
349
|
+
}
|
350
|
+
bok[attrs[:cfg_plural]] = new_resources
|
351
|
+
end
|
352
|
+
}
|
353
|
+
}
|
354
|
+
|
355
|
+
if save
|
356
|
+
MU.log "Committing adopted deployment to #{MU.dataDir}/deployments/#{deploy.deploy_id}", MU::NOTICE, details: origin
|
357
|
+
deploy.save!(force: true, origin: origin)
|
358
|
+
end
|
359
|
+
|
360
|
+
bok
|
361
|
+
end
|
362
|
+
|
363
|
+
def resolveReferences(cfg, deploy, parent)
|
364
|
+
if cfg.is_a?(MU::Config::Ref)
|
365
|
+
hashcfg = cfg.to_h
|
366
|
+
if cfg.kitten(deploy)
|
367
|
+
littermate = deploy.findLitterMate(type: cfg.type, name: cfg.name, cloud_id: cfg.id, habitat: cfg.habitat)
|
368
|
+
if littermate and littermate.config['name']
|
369
|
+
hashcfg['name'] = littermate.config['name']
|
370
|
+
hashcfg.delete("id") if hashcfg["name"]
|
371
|
+
hashcfg
|
372
|
+
elsif cfg.deploy_id and cfg.name and @savedeploys
|
373
|
+
hashcfg.delete("id") if hashcfg["name"]
|
374
|
+
hashcfg
|
375
|
+
elsif cfg.id
|
376
|
+
littermate = deploy.findLitterMate(type: cfg.type, cloud_id: cfg.id, habitat: cfg.habitat)
|
377
|
+
if littermate and littermate.config['name']
|
378
|
+
hashcfg['name'] = littermate.config['name']
|
379
|
+
hashcfg.delete("id") if hashcfg["name"]
|
380
|
+
elsif !@savedeploys
|
381
|
+
hashcfg.delete("deploy_id")
|
382
|
+
hashcfg.delete("name")
|
383
|
+
else
|
384
|
+
hashcfg.delete("name") if cfg.id and !cfg.deploy_id
|
385
|
+
end
|
386
|
+
end
|
387
|
+
elsif hashcfg["id"] # reference to raw cloud ids is reasonable
|
388
|
+
hashcfg.delete("deploy_id")
|
389
|
+
hashcfg.delete("name")
|
390
|
+
else
|
391
|
+
pp parent.cloud_desc
|
392
|
+
raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
|
393
|
+
end
|
394
|
+
hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id
|
395
|
+
cfg = hashcfg
|
396
|
+
elsif cfg.is_a?(Hash)
|
397
|
+
deletia = []
|
398
|
+
cfg.each_pair { |key, value|
|
399
|
+
begin
|
400
|
+
cfg[key] = resolveReferences(value, deploy, parent)
|
401
|
+
rescue Incomplete
|
402
|
+
MU.log "Dropping unresolved key #{key}", MU::WARN, details: cfg
|
403
|
+
deletia << key
|
404
|
+
end
|
405
|
+
}
|
406
|
+
deletia.each { |key|
|
407
|
+
cfg.delete(key)
|
408
|
+
}
|
409
|
+
cfg = nil if cfg.empty? and deletia.size > 0
|
410
|
+
elsif cfg.is_a?(Array)
|
411
|
+
new_array = []
|
412
|
+
cfg.each { |value|
|
413
|
+
begin
|
414
|
+
new_item = resolveReferences(value, deploy, parent)
|
415
|
+
if !new_item
|
416
|
+
MU.log "Dropping unresolved value", MU::WARN, details: value
|
417
|
+
else
|
418
|
+
new_array << new_item
|
419
|
+
end
|
420
|
+
rescue Incomplete
|
421
|
+
MU.log "Dropping unresolved value", MU::WARN, details: value
|
422
|
+
end
|
423
|
+
}
|
424
|
+
cfg = new_array
|
425
|
+
end
|
426
|
+
|
427
|
+
cfg
|
428
|
+
end
|
429
|
+
|
430
|
+
# @return [MU::MommaCat]
|
431
|
+
def generateStubDeploy(bok)
|
432
|
+
# hashify Ref objects before passing into here... or do we...?
|
433
|
+
|
434
|
+
time = Time.new
|
435
|
+
timestamp = time.strftime("%Y%m%d%H").to_s;
|
436
|
+
timestamp.freeze
|
437
|
+
|
438
|
+
retries = 0
|
439
|
+
deploy_id = nil
|
440
|
+
seed = nil
|
441
|
+
begin
|
442
|
+
raise MuError, "Failed to allocate an unused MU-ID after #{retries} tries!" if retries > 70
|
443
|
+
seedsize = 1 + (retries/10).abs
|
444
|
+
seed = (0...seedsize+1).map { ('a'..'z').to_a[rand(26)] }.join
|
445
|
+
deploy_id = bok['appname'].upcase + "-ADOPT-" + timestamp + "-" + seed.upcase
|
446
|
+
end while MU::MommaCat.deploy_exists?(deploy_id) or seed == "mu" or seed[0] == seed[1]
|
447
|
+
|
448
|
+
MU.setVar("deploy_id", deploy_id)
|
449
|
+
MU.setVar("appname", bok['appname'].upcase)
|
450
|
+
MU.setVar("environment", "ADOPT")
|
451
|
+
MU.setVar("timestamp", timestamp)
|
452
|
+
MU.setVar("seed", seed)
|
453
|
+
MU.setVar("handle", MU::MommaCat.generateHandle(seed))
|
454
|
+
|
455
|
+
deploy = MU::MommaCat.new(
|
456
|
+
deploy_id,
|
457
|
+
create: true,
|
458
|
+
config: bok,
|
459
|
+
environment: "adopt",
|
460
|
+
appname: bok['appname'].upcase,
|
461
|
+
timestamp: timestamp,
|
462
|
+
nocleanup: true,
|
463
|
+
no_artifacts: !(@savedeploys),
|
464
|
+
set_context_to_me: true,
|
465
|
+
mu_user: MU.mu_user
|
466
|
+
)
|
467
|
+
|
468
|
+
MU::Cloud.resource_types.each_pair { |typename, attrs|
|
469
|
+
if bok[attrs[:cfg_plural]]
|
470
|
+
bok[attrs[:cfg_plural]].each { |kitten|
|
471
|
+
|
472
|
+
if !@scraped[typename][kitten['cloud_id']]
|
473
|
+
MU.log "No object in scraped tree for #{attrs[:cfg_name]} #{kitten['cloud_id']} (#{kitten['name']})", MU::ERR, details: kitten
|
474
|
+
next
|
475
|
+
end
|
476
|
+
|
477
|
+
MU.log "Inserting #{attrs[:cfg_name]} #{kitten['name']} (#{kitten['cloud_id']}) into stub deploy", MU::DEBUG, details: @scraped[typename][kitten['cloud_id']]
|
478
|
+
|
479
|
+
@scraped[typename][kitten['cloud_id']].config!(kitten)
|
480
|
+
|
481
|
+
deploy.addKitten(
|
482
|
+
attrs[:cfg_plural],
|
483
|
+
kitten['name'],
|
484
|
+
@scraped[typename][kitten['cloud_id']]
|
485
|
+
)
|
486
|
+
}
|
487
|
+
end
|
488
|
+
}
|
489
|
+
|
490
|
+
deploy
|
491
|
+
end
|
492
|
+
|
493
|
+
# Go through everything we've scraped and update our mappings of cloud ids
|
494
|
+
# and bare name fields, so that resources can reference one another
|
495
|
+
# portably by name.
|
496
|
+
def catalogResources
|
497
|
+
end
|
498
|
+
|
499
|
+
end
|
500
|
+
end
|