cloud-mu 3.1.6 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. checksums.yaml +4 -4
  2. data/bin/mu-adopt +4 -12
  3. data/bin/mu-azure-tests +57 -0
  4. data/bin/mu-cleanup +2 -4
  5. data/bin/mu-configure +37 -1
  6. data/bin/mu-deploy +3 -3
  7. data/bin/mu-findstray-tests +25 -0
  8. data/bin/mu-gen-docs +2 -4
  9. data/bin/mu-run-tests +23 -10
  10. data/cloud-mu.gemspec +2 -2
  11. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  12. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  13. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  14. data/extras/generate-stock-images +1 -0
  15. data/modules/mu.rb +82 -95
  16. data/modules/mu/adoption.rb +356 -56
  17. data/modules/mu/cleanup.rb +21 -20
  18. data/modules/mu/cloud.rb +79 -1753
  19. data/modules/mu/cloud/database.rb +49 -0
  20. data/modules/mu/cloud/dnszone.rb +46 -0
  21. data/modules/mu/cloud/machine_images.rb +212 -0
  22. data/modules/mu/cloud/providers.rb +81 -0
  23. data/modules/mu/cloud/resource_base.rb +920 -0
  24. data/modules/mu/cloud/server.rb +40 -0
  25. data/modules/mu/cloud/server_pool.rb +1 -0
  26. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  27. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  28. data/modules/mu/cloud/wrappers.rb +165 -0
  29. data/modules/mu/config.rb +122 -80
  30. data/modules/mu/config/alarm.rb +2 -6
  31. data/modules/mu/config/bucket.rb +1 -1
  32. data/modules/mu/config/cache_cluster.rb +1 -1
  33. data/modules/mu/config/collection.rb +1 -1
  34. data/modules/mu/config/container_cluster.rb +2 -2
  35. data/modules/mu/config/database.rb +83 -104
  36. data/modules/mu/config/database.yml +1 -2
  37. data/modules/mu/config/dnszone.rb +1 -1
  38. data/modules/mu/config/doc_helpers.rb +4 -5
  39. data/modules/mu/config/endpoint.rb +1 -1
  40. data/modules/mu/config/firewall_rule.rb +3 -19
  41. data/modules/mu/config/folder.rb +1 -1
  42. data/modules/mu/config/function.rb +1 -1
  43. data/modules/mu/config/group.rb +1 -1
  44. data/modules/mu/config/habitat.rb +1 -1
  45. data/modules/mu/config/loadbalancer.rb +57 -11
  46. data/modules/mu/config/log.rb +1 -1
  47. data/modules/mu/config/msg_queue.rb +1 -1
  48. data/modules/mu/config/nosqldb.rb +1 -1
  49. data/modules/mu/config/notifier.rb +1 -1
  50. data/modules/mu/config/ref.rb +30 -4
  51. data/modules/mu/config/role.rb +1 -1
  52. data/modules/mu/config/schema_helpers.rb +30 -34
  53. data/modules/mu/config/search_domain.rb +1 -1
  54. data/modules/mu/config/server.rb +4 -12
  55. data/modules/mu/config/server_pool.rb +3 -7
  56. data/modules/mu/config/storage_pool.rb +1 -1
  57. data/modules/mu/config/tail.rb +10 -0
  58. data/modules/mu/config/user.rb +1 -1
  59. data/modules/mu/config/vpc.rb +12 -17
  60. data/modules/mu/defaults/AWS.yaml +32 -32
  61. data/modules/mu/defaults/Azure.yaml +1 -0
  62. data/modules/mu/defaults/Google.yaml +1 -0
  63. data/modules/mu/deploy.rb +16 -15
  64. data/modules/mu/groomer.rb +15 -0
  65. data/modules/mu/groomers/chef.rb +3 -0
  66. data/modules/mu/logger.rb +120 -144
  67. data/modules/mu/master.rb +1 -1
  68. data/modules/mu/mommacat.rb +54 -25
  69. data/modules/mu/mommacat/daemon.rb +10 -7
  70. data/modules/mu/mommacat/naming.rb +82 -3
  71. data/modules/mu/mommacat/search.rb +47 -15
  72. data/modules/mu/mommacat/storage.rb +72 -41
  73. data/modules/mu/{clouds → providers}/README.md +1 -1
  74. data/modules/mu/{clouds → providers}/aws.rb +114 -47
  75. data/modules/mu/{clouds → providers}/aws/alarm.rb +1 -1
  76. data/modules/mu/{clouds → providers}/aws/bucket.rb +2 -2
  77. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +10 -46
  78. data/modules/mu/{clouds → providers}/aws/collection.rb +3 -3
  79. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +15 -33
  80. data/modules/mu/providers/aws/database.rb +1744 -0
  81. data/modules/mu/{clouds → providers}/aws/dnszone.rb +2 -5
  82. data/modules/mu/{clouds → providers}/aws/endpoint.rb +2 -11
  83. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +33 -29
  84. data/modules/mu/{clouds → providers}/aws/folder.rb +0 -0
  85. data/modules/mu/{clouds → providers}/aws/function.rb +2 -10
  86. data/modules/mu/{clouds → providers}/aws/group.rb +9 -13
  87. data/modules/mu/{clouds → providers}/aws/habitat.rb +1 -1
  88. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +41 -33
  89. data/modules/mu/{clouds → providers}/aws/log.rb +2 -2
  90. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +2 -8
  91. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +0 -0
  92. data/modules/mu/{clouds → providers}/aws/notifier.rb +0 -0
  93. data/modules/mu/{clouds → providers}/aws/role.rb +7 -7
  94. data/modules/mu/{clouds → providers}/aws/search_domain.rb +8 -13
  95. data/modules/mu/{clouds → providers}/aws/server.rb +55 -90
  96. data/modules/mu/{clouds → providers}/aws/server_pool.rb +10 -33
  97. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +19 -36
  98. data/modules/mu/{clouds → providers}/aws/user.rb +8 -12
  99. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  100. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  101. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  102. data/modules/mu/{clouds → providers}/aws/vpc.rb +135 -70
  103. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  104. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  105. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  106. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  107. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  108. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  109. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  110. data/modules/mu/{clouds → providers}/azure/server.rb +30 -23
  111. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  112. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  113. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  114. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  115. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  116. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  117. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  118. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  119. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  120. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  121. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  122. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  123. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  124. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  125. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  126. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  127. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  128. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  129. data/modules/mu/{clouds → providers}/google.rb +14 -6
  130. data/modules/mu/{clouds → providers}/google/bucket.rb +1 -1
  131. data/modules/mu/{clouds → providers}/google/container_cluster.rb +28 -13
  132. data/modules/mu/{clouds → providers}/google/database.rb +1 -8
  133. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +2 -2
  134. data/modules/mu/{clouds → providers}/google/folder.rb +4 -8
  135. data/modules/mu/{clouds → providers}/google/function.rb +3 -3
  136. data/modules/mu/{clouds → providers}/google/group.rb +8 -16
  137. data/modules/mu/{clouds → providers}/google/habitat.rb +3 -7
  138. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +1 -1
  139. data/modules/mu/{clouds → providers}/google/role.rb +42 -34
  140. data/modules/mu/{clouds → providers}/google/server.rb +25 -10
  141. data/modules/mu/{clouds → providers}/google/server_pool.rb +10 -10
  142. data/modules/mu/{clouds → providers}/google/user.rb +31 -21
  143. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  144. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  145. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  146. data/modules/mu/{clouds → providers}/google/vpc.rb +37 -2
  147. data/modules/tests/centos6.yaml +11 -0
  148. data/modules/tests/centos7.yaml +11 -0
  149. data/modules/tests/centos8.yaml +12 -0
  150. data/modules/tests/rds.yaml +108 -0
  151. data/modules/tests/regrooms/rds.yaml +123 -0
  152. data/spec/mu/clouds/azure_spec.rb +2 -2
  153. metadata +108 -89
  154. data/modules/mu/clouds/aws/database.rb +0 -1974
@@ -0,0 +1,49 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic methods for all Database implementations
21
+ class Database
22
+
23
+ # Getting the password for a database's master user, and saving it in a database / cluster specific vault
24
+ def getPassword
25
+ if @config['password'].nil?
26
+ if @config['auth_vault'] && !@config['auth_vault'].empty?
27
+ @config['password'] = @groomclass.getSecret(
28
+ vault: @config['auth_vault']['vault'],
29
+ item: @config['auth_vault']['item'],
30
+ field: @config['auth_vault']['password_field']
31
+ )
32
+ else
33
+ # Should we use random instead?
34
+ @config['password'] = Password.pronounceable(10..12)
35
+ end
36
+ end
37
+
38
+ creds = {
39
+ "username" => @config["master_user"],
40
+ "password" => @config["password"]
41
+ }
42
+ @groomclass.saveSecret(vault: @mu_name, item: "database_credentials", data: creds)
43
+ end
44
+
45
+ end
46
+
47
+ end
48
+
49
+ end
@@ -0,0 +1,46 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic methods for all DNSZone implementations
21
+ class DNSZone
22
+
23
+ # Set a generic .platform-mu DNS entry for a resource, and return the name
24
+ # that was set.
25
+ def self.genericMuDNSEntry(*flags)
26
+ # XXX have this switch on a global config for where Mu puts its DNS
27
+ MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone").genericMuDNSEntry(flags.first)
28
+ end
29
+
30
+ # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all
31
+ # requested records in background and returns immediately.
32
+ # @param cfg [Array]: An array of parsed {MU::Config::BasketofKittens::dnszones::records} objects.
33
+ # @param target [String]: Optional target for the records to be created. Overrides targets embedded in cfg records.
34
+ def self.createRecordsFromConfig(*flags)
35
+ cloudclass = MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone")
36
+ if !flags.nil? and flags.size == 1
37
+ cloudclass.createRecordsFromConfig(flags.first)
38
+ else
39
+ cloudclass.createRecordsFromConfig(*flags)
40
+ end
41
+ end
42
+ end
43
+
44
+ end
45
+
46
+ end
@@ -0,0 +1,212 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # The public AWS S3 bucket where we expect to find YAML files listing our
21
+ # standard base images for various platforms.
22
+ BASE_IMAGE_BUCKET = "cloudamatic"
23
+ # The path in the AWS S3 bucket where we expect to find YAML files listing
24
+ # our standard base images for various platforms.
25
+ BASE_IMAGE_PATH = "/images"
26
+
27
+ # Aliases for platform names, in case we don't have actual images built for
28
+ # them.
29
+ PLATFORM_ALIASES = {
30
+ "linux" => "centos7",
31
+ "windows" => "win2k12r2",
32
+ "win2k12" => "win2k12r2",
33
+ "ubuntu" => "ubuntu16",
34
+ "centos" => "centos7",
35
+ "rhel7" => "rhel71",
36
+ "rhel" => "rhel71",
37
+ "amazon" => "amazon2016"
38
+ }
39
+
40
+ @@image_fetch_cache = {}
41
+ @@platform_cache = []
42
+ @@image_fetch_semaphore = Mutex.new
43
+
44
+ # Rifle our image lists from {MU::Cloud.getStockImage} and return a list
45
+ # of valid +platform+ names.
46
+ # @return [Array<String>]
47
+ def self.listPlatforms
48
+ return @@platform_cache if @@platform_cache and !@@platform_cache.empty?
49
+ @@platform_cache = MU::Cloud.supportedClouds.map { |cloud|
50
+ begin
51
+ resourceClass(cloud, :Server)
52
+ rescue MU::Cloud::MuCloudResourceNotImplemented, MU::MuError
53
+ next
54
+ end
55
+
56
+ images = MU::Cloud.getStockImage(cloud, quiet: true)
57
+ if images
58
+ images.keys
59
+ else
60
+ nil
61
+ end
62
+ }.flatten.uniq
63
+ @@platform_cache.delete(nil)
64
+ @@platform_cache.sort
65
+ @@platform_cache
66
+ end
67
+
68
+ # Locate a base image for a {MU::Cloud::Server} resource. First we check
69
+ # Mu's public bucket, which should list the latest and greatest. If we can't
70
+ # fetch that, then we fall back to a YAML file that's bundled as part of Mu,
71
+ # but which will typically be less up-to-date.
72
+ # @param cloud [String]: The cloud provider for which to return an image list
73
+ # @param platform [String]: The supported platform for which to return an image or images. If not specified, we'll return our entire library for the appropriate cloud provider.
74
+ # @param region [String]: The region for which the returned image or images should be supported, for cloud providers which require it (such as AWS).
75
+ # @param fail_hard [Boolean]: Raise an exception on most errors, such as an inability to reach our public listing, lack of matching images, etc.
76
+ # @return [Hash,String,nil]
77
+ def self.getStockImage(cloud = MU::Config.defaultCloud, platform: nil, region: nil, fail_hard: false, quiet: false)
78
+
79
+ if !MU::Cloud.supportedClouds.include?(cloud)
80
+ MU.log "'#{cloud}' is not a supported cloud provider! Available providers:", MU::ERR, details: MU::Cloud.supportedClouds
81
+ raise MuError, "'#{cloud}' is not a supported cloud provider!"
82
+ end
83
+
84
+ urls = ["http://"+BASE_IMAGE_BUCKET+".s3-website-us-east-1.amazonaws.com"+BASE_IMAGE_PATH]
85
+ if $MU_CFG and $MU_CFG['custom_images_url']
86
+ urls << $MU_CFG['custom_images_url']
87
+ end
88
+
89
+ images = nil
90
+ urls.each { |base_url|
91
+ @@image_fetch_semaphore.synchronize {
92
+ if @@image_fetch_cache[cloud] and (Time.now - @@image_fetch_cache[cloud]['time']) < 30
93
+ images = @@image_fetch_cache[cloud]['contents'].dup
94
+ else
95
+ begin
96
+ Timeout.timeout(2) do
97
+ response = open("#{base_url}/#{cloud}.yaml").read
98
+ images ||= {}
99
+ images.deep_merge!(YAML.load(response))
100
+ break
101
+ end
102
+ rescue StandardError => e
103
+ if fail_hard
104
+ raise MuError, "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})"
105
+ else
106
+ MU.log "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})", MU::WARN if !quiet
107
+ end
108
+ end
109
+ end
110
+ }
111
+ }
112
+
113
+ @@image_fetch_semaphore.synchronize {
114
+ @@image_fetch_cache[cloud] = {
115
+ 'contents' => images.dup,
116
+ 'time' => Time.now
117
+ }
118
+ }
119
+
120
+ backwards_compat = {
121
+ "AWS" => "amazon_images",
122
+ "Google" => "google_images",
123
+ }
124
+
125
+ # Load from inside our repository, if we didn't get images elsewise
126
+ if images.nil?
127
+ [backwards_compat[cloud], cloud].each { |file|
128
+ next if file.nil?
129
+ if File.exist?("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml")
130
+ images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml"))
131
+ break
132
+ end
133
+ }
134
+ end
135
+
136
+ # Now overlay local overrides, both of the systemwide (/opt/mu/etc) and
137
+ # per-user (~/.mu/etc) variety.
138
+ [backwards_compat[cloud], cloud].each { |file|
139
+ next if file.nil?
140
+ if File.exist?("#{MU.etcDir}/#{file}.yaml")
141
+ images ||= {}
142
+ images.deep_merge!(YAML.load(File.read("#{MU.etcDir}/#{file}.yaml")))
143
+ end
144
+ if Process.uid != 0
145
+ basepath = Etc.getpwuid(Process.uid).dir+"/.mu/etc"
146
+ if File.exist?("#{basepath}/#{file}.yaml")
147
+ images ||= {}
148
+ images.deep_merge!(YAML.load(File.read("#{basepath}/#{file}.yaml")))
149
+ end
150
+ end
151
+ }
152
+
153
+ if images.nil?
154
+ if fail_hard
155
+ raise MuError, "Failed to find any base images for #{cloud}"
156
+ else
157
+ MU.log "Failed to find any base images for #{cloud}", MU::WARN if !quiet
158
+ return nil
159
+ end
160
+ end
161
+
162
+ PLATFORM_ALIASES.each_pair { |a, t|
163
+ if images[t] and !images[a]
164
+ images[a] = images[t]
165
+ end
166
+ }
167
+
168
+ if platform
169
+ if !images[platform]
170
+ if fail_hard
171
+ raise MuError, "No base image for platform #{platform} in cloud #{cloud}"
172
+ else
173
+ MU.log "No base image for platform #{platform} in cloud #{cloud}", MU::WARN if !quiet
174
+ return nil
175
+ end
176
+ end
177
+ images = images[platform]
178
+
179
+ if region
180
+ # We won't fuss about the region argument if this isn't a cloud that
181
+ # has regions, just quietly don't bother.
182
+ if images.is_a?(Hash)
183
+ if images[region]
184
+ images = images[region]
185
+ else
186
+ if fail_hard
187
+ raise MuError, "No base image for platform #{platform} in cloud #{cloud} region #{region} found"
188
+ else
189
+ MU.log "No base image for platform #{platform} in cloud #{cloud} region #{region} found", MU::WARN if !quiet
190
+ return nil
191
+ end
192
+ end
193
+ end
194
+ end
195
+ else
196
+ if region
197
+ images.values.each { |regions|
198
+ # Filter to match our requested region, but for all the platforms,
199
+ # since we didn't specify one.
200
+ if regions.is_a?(Hash)
201
+ regions.delete_if { |r| r != region }
202
+ end
203
+ }
204
+ end
205
+ end
206
+
207
+ images
208
+ end
209
+
210
+ end
211
+
212
+ end
@@ -0,0 +1,81 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # List of known/supported Cloud providers. This may be modified at runtime
21
+ # if an implemention is defective or missing required methods.
22
+ @@supportedCloudList = ['AWS', 'CloudFormation', 'Google', 'Azure']
23
+
24
+ # List of known/supported Cloud providers
25
+ # @return [Array<String>]
26
+ def self.supportedClouds
27
+ @@supportedCloudList
28
+ end
29
+
30
+ # Raise an exception if the cloud provider specified isn't valid
31
+ def self.cloudClass(cloud)
32
+ if cloud.nil? or !supportedClouds.include?(cloud.to_s)
33
+ raise MuError, "Cloud provider #{cloud} is not supported"
34
+ end
35
+ Object.const_get("MU").const_get("Cloud").const_get(cloud.to_s)
36
+ end
37
+
38
+ # List of known/supported Cloud providers for which we have at least one
39
+ # set of credentials configured.
40
+ # @return [Array<String>]
41
+ def self.availableClouds
42
+ available = []
43
+ MU::Cloud.supportedClouds.each { |cloud|
44
+ begin
45
+ cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
46
+ next if cloudbase.listCredentials.nil? or cloudbase.listCredentials.empty?
47
+ available << cloud
48
+ rescue NameError
49
+ end
50
+ }
51
+
52
+ available
53
+ end
54
+
55
+ # Raise an exception if the cloud provider specified isn't valid or we
56
+ # don't have any credentials configured for it.
57
+ def self.assertAvailableCloud(cloud)
58
+ if cloud.nil? or availableClouds.include?(cloud.to_s)
59
+ raise MuError, "Cloud provider #{cloud} is not available"
60
+ end
61
+ end
62
+
63
+ # Load the container class for each cloud we know about, and inject autoload
64
+ # code for each of its supported resource type classes.
65
+ failed = []
66
+ MU::Cloud.supportedClouds.each { |cloud|
67
+ require "mu/providers/#{cloud.downcase}"
68
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
69
+ @@generic_class_methods_toplevel.each { |method|
70
+ if !cloudclass.respond_to?(method)
71
+ MU.log "MU::Cloud::#{cloud} has not implemented required class method #{method}, disabling", MU::ERR
72
+ failed << cloud
73
+ end
74
+ }
75
+ }
76
+ failed.uniq!
77
+ @@supportedCloudList = @@supportedCloudList - failed
78
+
79
+ end
80
+
81
+ end
@@ -0,0 +1,920 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic class methods (.find, .cleanup, etc) are defined in wrappers.rb
21
+ require 'mu/cloud/wrappers'
22
+
23
+ @@resource_types.each_key { |name|
24
+ Object.const_get("MU").const_get("Cloud").const_get(name).class_eval {
25
+ attr_reader :cloudclass
26
+ attr_reader :cloudobj
27
+ attr_reader :destroyed
28
+ attr_reader :delayed_save
29
+
30
+ # Print something palatable when we're called in a string context.
31
+ def to_s
32
+ fullname = "#{self.class.shortname}"
33
+ if !@cloudobj.nil? and !@cloudobj.mu_name.nil?
34
+ @mu_name ||= @cloudobj.mu_name
35
+ end
36
+ if !@mu_name.nil? and !@mu_name.empty?
37
+ fullname = fullname + " '#{@mu_name}'"
38
+ end
39
+ if !@cloud_id.nil?
40
+ fullname = fullname + " (#{@cloud_id})"
41
+ end
42
+ return fullname
43
+ end
44
+
45
+ # Set our +deploy+ and +deploy_id+ attributes, optionally doing so even
46
+ # if they have already been set.
47
+ #
48
+ # @param mommacat [MU::MommaCat]: The deploy to which we're being told we belong
49
+ # @param force [Boolean]: Set even if we already have a deploy object
50
+ # @return [String]: Our new +deploy_id+
51
+ def intoDeploy(mommacat, force: false)
52
+ if force or (!@deploy)
53
+ MU.log "Inserting #{self} [#{self.object_id}] into #{mommacat.deploy_id} as a #{@config['name']}", MU::DEBUG
54
+
55
+ @deploy = mommacat
56
+ @deploy.addKitten(@cloudclass.cfg_plural, @config['name'], self)
57
+ @deploy_id = @deploy.deploy_id
58
+ @cloudobj.intoDeploy(mommacat, force: force) if @cloudobj
59
+ end
60
+ @deploy_id
61
+ end
62
+
63
+ # Return the +virtual_name+ config field, if it is set.
64
+ # @param name [String]: If set, will only return a value if +virtual_name+ matches this string
65
+ # @return [String,nil]
66
+ def virtual_name(name = nil)
67
+ if @config and @config['virtual_name'] and
68
+ (!name or name == @config['virtual_name'])
69
+ return @config['virtual_name']
70
+ end
71
+ nil
72
+ end
73
+
74
+ # @param mommacat [MU::MommaCat]: The deployment containing this cloud resource
75
+ # @param mu_name [String]: Optional- specify the full Mu resource name of an existing resource to load, instead of creating a new one
76
+ # @param cloud_id [String]: Optional- specify the cloud provider's identifier for an existing resource to load, instead of creating a new one
77
+ # @param kitten_cfg [Hash]: The parse configuration for this object from {MU::Config}
78
+ def initialize(**args)
79
+ raise MuError, "Cannot invoke Cloud objects without a configuration" if args[:kitten_cfg].nil?
80
+
81
+ # We are a parent wrapper object. Initialize our child object and
82
+ # housekeeping bits accordingly.
83
+ if self.class.name =~ /^MU::Cloud::([^:]+)$/
84
+ @live = true
85
+ @delayed_save = args[:delayed_save]
86
+ @method_semaphore = Mutex.new
87
+ @method_locks = {}
88
+ if args[:mommacat]
89
+ MU.log "Initializing an instance of #{self.class.name} in #{args[:mommacat].deploy_id} #{mu_name}", MU::DEBUG, details: args[:kitten_cfg]
90
+ elsif args[:mu_name].nil?
91
+ raise MuError, "Can't instantiate a MU::Cloud object with a live deploy or giving us a mu_name"
92
+ else
93
+ MU.log "Initializing a detached #{self.class.name} named #{args[:mu_name]}", MU::DEBUG, details: args[:kitten_cfg]
94
+ end
95
+
96
+ my_cloud = args[:kitten_cfg]['cloud'].to_s || MU::Config.defaultCloud
97
+ if (my_cloud.nil? or my_cloud.empty?) and args[:mommacat]
98
+ my_cloud = args[:mommacat].original_config['cloud']
99
+ end
100
+ if my_cloud.nil? or !MU::Cloud.supportedClouds.include?(my_cloud)
101
+ raise MuError, "Can't instantiate a MU::Cloud object without a valid cloud (saw '#{my_cloud}')"
102
+ end
103
+ @cloudclass = MU::Cloud.resourceClass(my_cloud, self.class.shortname)
104
+ @cloud_desc_cache ||= args[:from_cloud_desc] if args[:from_cloud_desc]
105
+ @cloudparentclass = MU::Cloud.cloudClass(my_cloud)
106
+ @cloudobj = @cloudclass.new(
107
+ mommacat: args[:mommacat],
108
+ kitten_cfg: args[:kitten_cfg],
109
+ cloud_id: args[:cloud_id],
110
+ mu_name: args[:mu_name]
111
+ )
112
+ raise MuError, "Unknown error instantiating #{self}" if @cloudobj.nil?
113
+ # These should actually call the method live instead of caching a static value
114
+ PUBLIC_ATTRS.each { |a|
115
+ begin
116
+ instance_variable_set(("@"+a.to_s).to_sym, @cloudobj.send(a))
117
+ rescue NoMethodError => e
118
+ MU.log "#{@cloudclass.name} failed to implement method '#{a}'", MU::ERR, details: e.message
119
+ raise e
120
+ end
121
+ }
122
+ @deploy ||= args[:mommacat]
123
+ @deploy_id ||= @deploy.deploy_id if @deploy
124
+
125
+ # Register with the containing deployment
126
+ if !@deploy.nil? and !@cloudobj.mu_name.nil? and
127
+ !@cloudobj.mu_name.empty? and !args[:delay_descriptor_load]
128
+ describe # XXX is this actually safe here?
129
+ @deploy.addKitten(self.class.cfg_name, @config['name'], self)
130
+ elsif !@deploy.nil? and @cloudobj.mu_name.nil?
131
+ MU.log "#{self} in #{@deploy.deploy_id} didn't generate a mu_name after being loaded/initialized, dependencies on this resource will probably be confused!", MU::ERR, details: [caller, args.keys]
132
+ end
133
+
134
+ # We are actually a child object invoking this via super() from its
135
+ # own initialize(), so initialize all the attributes and instance
136
+ # variables we know to be universal.
137
+ else
138
+ class << self
139
+ # Declare attributes that everyone should have
140
+ PUBLIC_ATTRS.each { |a|
141
+ attr_reader a
142
+ }
143
+ end
144
+ # XXX this butchers ::Id and ::Ref objects that might be used by dependencies() to good effect, but we also can't expect our implementations to cope with knowing when a .to_s has to be appended to things at random
145
+ @config = MU::Config.manxify(args[:kitten_cfg]) || MU::Config.manxify(args[:config])
146
+
147
+ if !@config
148
+ MU.log "Missing config arguments in setInstanceVariables, can't initialize a cloud object without it", MU::ERR, details: args.keys
149
+ raise MuError, "Missing config arguments in setInstanceVariables"
150
+ end
151
+
152
+ @deploy = args[:mommacat] || args[:deploy]
153
+ @cloud_desc_cache ||= args[:from_cloud_desc] if args[:from_cloud_desc]
154
+
155
+ @credentials = args[:credentials]
156
+ @credentials ||= @config['credentials']
157
+
158
+ @cloud = @config['cloud']
159
+ if !@cloud
160
+ if self.class.name =~ /^MU::Cloud::([^:]+)(?:::.+|$)/
161
+ cloudclass_name = Regexp.last_match[1]
162
+ if MU::Cloud.supportedClouds.include?(cloudclass_name)
163
+ @cloud = cloudclass_name
164
+ end
165
+ end
166
+ end
167
+ if !@cloud
168
+ raise MuError, "Failed to determine what cloud #{self} should be in!"
169
+ end
170
+
171
+ @environment = @config['environment']
172
+ if @deploy
173
+ @deploy_id = @deploy.deploy_id
174
+ @appname = @deploy.appname
175
+ end
176
+
177
+ @cloudclass = MU::Cloud.resourceClass(@cloud, self.class.shortname)
178
+ @cloudparentclass = MU::Cloud.cloudClass(@cloud)
179
+
180
+ # A pre-existing object, you say?
181
+ if args[:cloud_id]
182
+
183
+ # TODO implement ::Id for every cloud... and they should know how to get from
184
+ # cloud_desc to a fully-resolved ::Id object, not just the short string
185
+
186
+ @cloud_id = args[:cloud_id]
187
+ describe(cloud_id: @cloud_id)
188
+ @habitat_id = habitat_id # effectively, cache this
189
+
190
+ # If we can build us an ::Id object for @cloud_id instead of a
191
+ # string, do so.
192
+ begin
193
+ idclass = @cloudparentclass.const_get(:Id)
194
+ long_id = if @deploydata and @deploydata[idclass.idattr.to_s]
195
+ @deploydata[idclass.idattr.to_s]
196
+ elsif self.respond_to?(idclass.idattr)
197
+ self.send(idclass.idattr)
198
+ end
199
+
200
+ @cloud_id = idclass.new(long_id) if !long_id.nil? and !long_id.empty?
201
+ # 1 see if we have the value on the object directly or in deploy data
202
+ # 2 set an attr_reader with the value
203
+ # 3 rewrite our @cloud_id attribute with a ::Id object
204
+ rescue NameError, MU::Cloud::MuCloudResourceNotImplemented
205
+ end
206
+
207
+ end
208
+
209
+ # Use pre-existing mu_name (we're probably loading an extant deploy)
210
+ # if available
211
+ if args[:mu_name]
212
+ @mu_name = args[:mu_name].dup
213
+ # If scrub_mu_isms is set, our mu_name is always just the bare name
214
+ # field of the resource.
215
+ elsif @config['scrub_mu_isms']
216
+ @mu_name = @config['name'].dup
217
+ # XXX feck it insert an inheritable method right here? Set a default? How should resource implementations determine whether they're instantiating a new object?
218
+ end
219
+
220
+ @tags = {}
221
+ if !@config['scrub_mu_isms']
222
+ @tags = @deploy ? @deploy.listStandardTags : MU::MommaCat.listStandardTags
223
+ end
224
+ if @config['tags']
225
+ @config['tags'].each { |tag|
226
+ @tags[tag['key']] = tag['value']
227
+ }
228
+ end
229
+
230
+ if @cloudparentclass.respond_to?(:resourceInitHook)
231
+ @cloudparentclass.resourceInitHook(self, @deploy)
232
+ end
233
+
234
+ # Add cloud-specific instance methods for our resource objects to
235
+ # inherit.
236
+ if @cloudparentclass.const_defined?(:AdditionalResourceMethods)
237
+ self.extend @cloudparentclass.const_get(:AdditionalResourceMethods)
238
+ end
239
+
240
+ if ["Server", "ServerPool"].include?(self.class.shortname) and @deploy
241
+ @mu_name ||= @deploy.getResourceName(@config['name'], need_unique_string: @config.has_key?("basis"))
242
+ if self.class.shortname == "Server"
243
+ @groomer = MU::Groomer.new(self)
244
+ end
245
+
246
+ @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
247
+
248
+ if windows? or @config['active_directory'] and !@mu_windows_name
249
+ if !@deploydata.nil? and !@deploydata['mu_windows_name'].nil?
250
+ @mu_windows_name = @deploydata['mu_windows_name']
251
+ else
252
+ # Use the same random differentiator as the "real" name if we're
253
+ # from a ServerPool. Helpful for admin sanity.
254
+ unq = @mu_name.sub(/^.*?-(...)$/, '\1')
255
+ if @config['basis'] and !unq.nil? and !unq.empty?
256
+ @mu_windows_name = @deploy.getResourceName(@config['name'], max_length: 15, need_unique_string: true, use_unique_string: unq, reuse_unique_string: true)
257
+ else
258
+ @mu_windows_name = @deploy.getResourceName(@config['name'], max_length: 15, need_unique_string: true)
259
+ end
260
+ end
261
+ end
262
+ class << self
263
+ attr_reader :groomer
264
+ attr_reader :groomerclass
265
+ attr_accessor :mu_windows_name # XXX might be ok as reader now
266
+ end
267
+ end
268
+ end
269
+
270
+ end
271
+
272
+ def cloud
273
+ if @cloud
274
+ @cloud
275
+ elsif @config and @config['cloud']
276
+ @config['cloud']
277
+ elsif self.class.name =~ /^MU::Cloud::([^:]+)::.+/
278
+ cloudclass_name = Regexp.last_match[1]
279
+ if MU::Cloud.supportedClouds.include?(cloudclass_name)
280
+ cloudclass_name
281
+ else
282
+ nil
283
+ end
284
+ else
285
+ nil
286
+ end
287
+ end
288
+
289
+
290
+ # Remove all metadata and cloud resources associated with this object
291
+ def destroy
292
+ if self.class.cfg_name == "server"
293
+ begin
294
+ ip = canonicalIP
295
+ MU::Master.removeIPFromSSHKnownHosts(ip) if ip
296
+ if @deploy and @deploy.deployment and
297
+ @deploy.deployment['servers'] and @config['name']
298
+ me = @deploy.deployment['servers'][@config['name']][@mu_name]
299
+ if me
300
+ ["private_ip_address", "public_ip_address"].each { |field|
301
+ if me[field]
302
+ MU::Master.removeIPFromSSHKnownHosts(me[field])
303
+ end
304
+ }
305
+ if me["private_ip_list"]
306
+ me["private_ip_list"].each { |private_ip|
307
+ MU::Master.removeIPFromSSHKnownHosts(private_ip)
308
+ }
309
+ end
310
+ end
311
+ end
312
+ rescue MU::MuError => e
313
+ MU.log e.message, MU::WARN
314
+ end
315
+ end
316
+ if !@cloudobj.nil? and !@cloudobj.groomer.nil?
317
+ @cloudobj.groomer.cleanup
318
+ elsif !@groomer.nil?
319
+ @groomer.cleanup
320
+ end
321
+ if !@deploy.nil?
322
+ if !@cloudobj.nil? and !@config.nil? and !@cloudobj.mu_name.nil?
323
+ @deploy.notify(self.class.cfg_plural, @config['name'], nil, mu_name: @cloudobj.mu_name, remove: true, triggering_node: @cloudobj, delayed_save: @delayed_save)
324
+ elsif !@mu_name.nil?
325
+ @deploy.notify(self.class.cfg_plural, @config['name'], nil, mu_name: @mu_name, remove: true, triggering_node: self, delayed_save: @delayed_save)
326
+ end
327
+ @deploy.removeKitten(self)
328
+ end
329
+ # Make sure that if notify gets called again it won't go returning a
330
+ # bunch of now-bogus metadata.
331
+ @destroyed = true
332
+ if !@cloudobj.nil?
333
+ def @cloudobj.notify
334
+ {}
335
+ end
336
+ else
337
+ def notify
338
+ {}
339
+ end
340
+ end
341
+ end
342
+
343
+ # Return the cloud object's idea of where it lives (project, account,
344
+ # etc) in the form of an identifier. If not applicable for this object,
345
+ # we expect to return +nil+.
346
+ # @return [String,nil]
347
+ def habitat(nolookup: true)
348
+ return nil if ["folder", "habitat"].include?(self.class.cfg_name)
349
+ if @cloudobj
350
+ @cloudparentclass.habitat(@cloudobj, nolookup: nolookup, deploy: @deploy)
351
+ else
352
+ @cloudparentclass.habitat(self, nolookup: nolookup, deploy: @deploy)
353
+ end
354
+ end
355
+
356
+ def habitat_id(nolookup: false)
357
+ @habitat_id ||= habitat(nolookup: nolookup)
358
+ @habitat_id
359
+ end
360
+
361
+ # We're fundamentally a wrapper class, so go ahead and reroute requests
362
+ # that are meant for our wrapped object.
363
+ def method_missing(method_sym, *arguments)
364
+ if @cloudobj
365
+ MU.log "INVOKING #{method_sym} FROM PARENT CLOUD OBJECT #{self}", MU::DEBUG, details: arguments
366
+ @cloudobj.method(method_sym).call(*arguments)
367
+ else
368
+ raise NoMethodError, "No such instance method #{method_sym} available on #{self.class.name}"
369
+ end
370
+ end
371
+
372
+ # Merge the passed hash into the existing configuration hash of this
373
+ # cloud object. Currently this is only used by the {MU::Adoption}
374
+ # module. I don't love exposing this to the whole internal API, but I'm
375
+ # probably overthinking that.
376
+ # @param newcfg [Hash]
377
+ def config!(newcfg)
378
+ @config.merge!(newcfg)
379
+ end
380
+
381
+ def cloud_desc(use_cache: true)
382
+ describe
383
+
384
+ if !@cloudobj.nil?
385
+ if @cloudobj.class.instance_methods(false).include?(:cloud_desc)
386
+ @cloud_desc_cache ||= @cloudobj.cloud_desc
387
+ end
388
+ end
389
+ if !@config.nil? and !@cloud_id.nil? and (!use_cache or @cloud_desc_cache.nil?)
390
+ # The find() method should be returning a Hash with the cloud_id
391
+ # as a key and a cloud platform descriptor as the value.
392
+ begin
393
+ args = {
394
+ :region => @config['region'],
395
+ :cloud => @config['cloud'],
396
+ :cloud_id => @cloud_id,
397
+ :credentials => @credentials,
398
+ :project => habitat_id, # XXX this belongs in our required_instance_methods hack
399
+ :flags => @config
400
+ }
401
+ @cloudparentclass.required_instance_methods.each { |m|
402
+ # if respond_to?(m)
403
+ # args[m] = method(m).call
404
+ # else
405
+ args[m] = instance_variable_get(("@"+m.to_s).to_sym)
406
+ # end
407
+ }
408
+
409
+ matches = self.class.find(args)
410
+ if !matches.nil? and matches.is_a?(Hash)
411
+ # XXX or if the hash is keyed with an ::Id element, oh boy
412
+ # puts matches[@cloud_id][:self_link]
413
+ # puts matches[@cloud_id][:url]
414
+ # if matches[@cloud_id][:self_link]
415
+ # @url ||= matches[@cloud_id][:self_link]
416
+ # elsif matches[@cloud_id][:url]
417
+ # @url ||= matches[@cloud_id][:url]
418
+ # elsif matches[@cloud_id][:arn]
419
+ # @arn ||= matches[@cloud_id][:arn]
420
+ # end
421
+ if matches[@cloud_id]
422
+ @cloud_desc_cache = matches[@cloud_id]
423
+ else
424
+ matches.each_pair { |k, v| # flatten out ::Id objects just in case
425
+ if @cloud_id.to_s == k.to_s
426
+ @cloud_desc_cache = v
427
+ break
428
+ end
429
+ }
430
+ end
431
+ end
432
+
433
+ if !@cloud_desc_cache
434
+ MU.log "cloud_desc via #{self.class.name}.find() failed to locate a live object.\nWas called by #{caller(1..1)}", MU::WARN, details: args
435
+ end
436
+ rescue StandardError => e
437
+ MU.log "Got #{e.inspect} trying to find cloud handle for #{self.class.shortname} #{@mu_name} (#{@cloud_id})", MU::WARN
438
+ raise e
439
+ end
440
+ end
441
+
442
+ return @cloud_desc_cache
443
+ end
444
+
445
+ # Retrieve all of the known metadata for this resource.
446
+ # @param cloud_id [String]: The cloud platform's identifier for the resource we're describing. Makes lookups more efficient.
447
+ # @return [Array<Hash>]: mu_name, config, deploydata
448
+ def describe(cloud_id: nil)
449
+ if cloud_id.nil? and !@cloudobj.nil?
450
+ @cloud_id ||= @cloudobj.cloud_id
451
+ end
452
+ res_type = self.class.cfg_plural
453
+ res_name = @config['name'] if !@config.nil?
454
+ @credentials ||= @config['credentials'] if !@config.nil?
455
+ deploydata = nil
456
+
457
+ if !@deploy.nil? and @deploy.is_a?(MU::MommaCat) and
458
+ !@deploy.deployment.nil? and
459
+ !@deploy.deployment[res_type].nil? and
460
+ !@deploy.deployment[res_type][res_name].nil?
461
+ deploydata = @deploy.deployment[res_type][res_name]
462
+ else
463
+ # XXX This should only happen on a brand new resource, but we should
464
+ # probably complain under other circumstances, if we can
465
+ # differentiate them.
466
+ end
467
+
468
+ if self.class.has_multiples and !@mu_name.nil? and deploydata.is_a?(Hash) and deploydata.has_key?(@mu_name)
469
+ @deploydata = deploydata[@mu_name]
470
+ elsif deploydata.is_a?(Hash)
471
+ @deploydata = deploydata
472
+ end
473
+
474
+ if @cloud_id.nil? and @deploydata.is_a?(Hash)
475
+ if @mu_name.nil? and @deploydata.has_key?('#MU_NAME')
476
+ @mu_name = @deploydata['#MU_NAME']
477
+ end
478
+ if @deploydata.has_key?('cloud_id')
479
+ @cloud_id ||= @deploydata['cloud_id']
480
+ end
481
+ end
482
+
483
+ return [@mu_name, @config, @deploydata]
484
+ end
485
+
486
+ # Fetch MU::Cloud objects for each of this object's dependencies, and
487
+ # return in an easily-navigable Hash. This can include things listed in
488
+ # @config['dependencies'], implicitly-defined dependencies such as
489
+ # add_firewall_rules or vpc stanzas, and may refer to objects internal
490
+ # to this deployment or external. Will populate the instance variables
491
+ # @dependencies (general dependencies, which can only be sibling
492
+ # resources in this deployment), as well as for certain config stanzas
493
+ # which can refer to external resources (@vpc, @loadbalancers,
494
+ # @add_firewall_rules)
495
+ def dependencies(use_cache: false, debug: false)
496
+ @dependencies ||= {}
497
+ @loadbalancers ||= []
498
+ @firewall_rules ||= []
499
+
500
+ if @config.nil?
501
+ return [@dependencies, @vpc, @loadbalancers]
502
+ end
503
+ if use_cache and @dependencies.size > 0
504
+ return [@dependencies, @vpc, @loadbalancers]
505
+ end
506
+ @config['dependencies'] = [] if @config['dependencies'].nil?
507
+
508
+ loglevel = debug ? MU::NOTICE : MU::DEBUG
509
+
510
+ # First, general dependencies. These should all be fellow members of
511
+ # the current deployment.
512
+ @config['dependencies'].each { |dep|
513
+ @dependencies[dep['type']] ||= {}
514
+ next if @dependencies[dep['type']].has_key?(dep['name'])
515
+ handle = @deploy.findLitterMate(type: dep['type'], name: dep['name']) if !@deploy.nil?
516
+ if !handle.nil?
517
+ MU.log "Loaded dependency for #{self}: #{dep['name']} => #{handle}", loglevel
518
+ @dependencies[dep['type']][dep['name']] = handle
519
+ else
520
+ # XXX yell under circumstances where we should expect to have
521
+ # our stuff available already?
522
+ end
523
+ }
524
+
525
+ # Special dependencies: my containing VPC
526
+ if self.class.can_live_in_vpc and !@config['vpc'].nil?
527
+ @config['vpc']["id"] ||= @config['vpc']["vpc_id"] # old deploys
528
+ @config['vpc']["name"] ||= @config['vpc']["vpc_name"] # old deploys
529
+ # If something hash-ified a MU::Config::Ref here, fix it
530
+ if !@config['vpc']["id"].nil? and @config['vpc']["id"].is_a?(Hash)
531
+ @config['vpc']["id"] = MU::Config::Ref.new(@config['vpc']["id"])
532
+ end
533
+ if !@config['vpc']["id"].nil?
534
+ if @config['vpc']["id"].is_a?(MU::Config::Ref) and !@config['vpc']["id"].kitten.nil?
535
+ @vpc = @config['vpc']["id"].kitten(@deploy)
536
+ else
537
+ if @config['vpc']['habitat']
538
+ @config['vpc']['habitat'] = MU::Config::Ref.get(@config['vpc']['habitat'])
539
+ end
540
+ vpc_ref = MU::Config::Ref.get(@config['vpc'])
541
+ @vpc = vpc_ref.kitten(@deploy)
542
+ end
543
+ elsif !@config['vpc']["name"].nil? and @deploy
544
+ MU.log "Attempting findLitterMate on VPC for #{self}", loglevel, details: @config['vpc']
545
+
546
+ sib_by_name = @deploy.findLitterMate(name: @config['vpc']['name'], type: "vpcs", return_all: true, habitat: @config['vpc']['project'], debug: debug)
547
+ if sib_by_name.is_a?(Hash)
548
+ if sib_by_name.size == 1
549
+ @vpc = sib_by_name.values.first
550
+ MU.log "Single VPC match for #{self}", loglevel, details: @vpc.to_s
551
+ else
552
+ # XXX ok but this is the wrong place for this really the config parser needs to sort this out somehow
553
+ # we got multiple matches, try to pick one by preferred subnet
554
+ # behavior
555
+ MU.log "Sorting a bunch of VPC matches for #{self}", loglevel, details: sib_by_name.map { |s| s.to_s }.join(", ")
556
+ sib_by_name.values.each { |sibling|
557
+ all_private = sibling.subnets.map { |s| s.private? }.all?(true)
558
+ all_public = sibling.subnets.map { |s| s.private? }.all?(false)
559
+ names = sibling.subnets.map { |s| s.name }
560
+ ids = sibling.subnets.map { |s| s.cloud_id }
561
+ if all_private and ["private", "all_private"].include?(@config['vpc']['subnet_pref'])
562
+ @vpc = sibling
563
+ break
564
+ elsif all_public and ["public", "all_public"].include?(@config['vpc']['subnet_pref'])
565
+ @vpc = sibling
566
+ break
567
+ elsif @config['vpc']['subnet_name'] and
568
+ names.include?(@config['vpc']['subnet_name'])
569
+ #puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
570
+ @vpc = sibling
571
+ break
572
+ elsif @config['vpc']['subnet_id'] and
573
+ ids.include?(@config['vpc']['subnet_id'])
574
+ @vpc = sibling
575
+ break
576
+ end
577
+ }
578
+ if !@vpc
579
+ sibling = sib_by_name.values.sample
580
+ MU.log "Got multiple matching VPCs for #{self.class.cfg_name} #{@mu_name}, so I'm arbitrarily choosing #{sibling.mu_name}", MU::WARN, details: @config['vpc']
581
+ @vpc = sibling
582
+ end
583
+ end
584
+ else
585
+ @vpc = sib_by_name
586
+ MU.log "Found exact VPC match for #{self}", loglevel, details: sib_by_name.to_s
587
+ end
588
+ else
589
+ MU.log "No shortcuts available to fetch VPC for #{self}", loglevel, details: @config['vpc']
590
+ end
591
+
592
+ if !@vpc and !@config['vpc']["name"].nil? and
593
+ @dependencies.has_key?("vpc") and
594
+ @dependencies["vpc"].has_key?(@config['vpc']["name"])
595
+ MU.log "Grabbing VPC I see in @dependencies['vpc']['#{@config['vpc']["name"]}'] for #{self}", loglevel, details: @config['vpc']
596
+ @vpc = @dependencies["vpc"][@config['vpc']["name"]]
597
+ elsif !@vpc
598
+ tag_key, tag_value = @config['vpc']['tag'].split(/=/, 2) if !@config['vpc']['tag'].nil?
599
+ if !@config['vpc'].has_key?("id") and
600
+ !@config['vpc'].has_key?("deploy_id") and !@deploy.nil?
601
+ @config['vpc']["deploy_id"] = @deploy.deploy_id
602
+ end
603
+ MU.log "Doing findStray for VPC for #{self}", loglevel, details: @config['vpc']
604
+ vpcs = MU::MommaCat.findStray(
605
+ @config['cloud'],
606
+ "vpc",
607
+ deploy_id: @config['vpc']["deploy_id"],
608
+ cloud_id: @config['vpc']["id"],
609
+ name: @config['vpc']["name"],
610
+ tag_key: tag_key,
611
+ tag_value: tag_value,
612
+ habitats: [@project_id],
613
+ region: @config['vpc']["region"],
614
+ calling_deploy: @deploy,
615
+ credentials: @credentials,
616
+ dummy_ok: true,
617
+ debug: debug
618
+ )
619
+ @vpc = vpcs.first if !vpcs.nil? and vpcs.size > 0
620
+ end
621
+ if @vpc and @vpc.config and @vpc.config['bastion'] and
622
+ @vpc.config['bastion'].to_h['name'] != @config['name']
623
+ refhash = @vpc.config['bastion'].to_h
624
+ refhash['deploy_id'] ||= @vpc.deploy.deploy_id
625
+ natref = MU::Config::Ref.get(refhash)
626
+ if natref and natref.kitten(@vpc.deploy)
627
+ @nat = natref.kitten(@vpc.deploy)
628
+ end
629
+ end
630
+ if @nat.nil? and !@vpc.nil? and (
631
+ @config['vpc'].has_key?("nat_host_id") or
632
+ @config['vpc'].has_key?("nat_host_tag") or
633
+ @config['vpc'].has_key?("nat_host_ip") or
634
+ @config['vpc'].has_key?("nat_host_name")
635
+ )
636
+
637
+ nat_tag_key, nat_tag_value = @config['vpc']['nat_host_tag'].split(/=/, 2) if !@config['vpc']['nat_host_tag'].nil?
638
+
639
+ @nat = @vpc.findBastion(
640
+ nat_name: @config['vpc']['nat_host_name'],
641
+ nat_cloud_id: @config['vpc']['nat_host_id'],
642
+ nat_tag_key: nat_tag_key,
643
+ nat_tag_value: nat_tag_value,
644
+ nat_ip: @config['vpc']['nat_host_ip']
645
+ )
646
+
647
+ if @nat.nil?
648
+ if !@vpc.cloud_desc.nil?
649
+ @nat = @vpc.findNat(
650
+ nat_cloud_id: @config['vpc']['nat_host_id'],
651
+ nat_filter_key: "vpc-id",
652
+ region: @config['vpc']["region"],
653
+ nat_filter_value: @vpc.cloud_id,
654
+ credentials: @config['credentials']
655
+ )
656
+ else
657
+ @nat = @vpc.findNat(
658
+ nat_cloud_id: @config['vpc']['nat_host_id'],
659
+ region: @config['vpc']["region"],
660
+ credentials: @config['credentials']
661
+ )
662
+ end
663
+ end
664
+ end
665
+ if @vpc.nil? and @config['vpc']
666
+ feck = MU::Config::Ref.get(@config['vpc'])
667
+ feck.kitten(@deploy, debug: true)
668
+ pp feck
669
+ raise MuError.new "#{self.class.cfg_name} #{@config['name']} failed to locate its VPC", details: @config['vpc']
670
+ end
671
+ elsif self.class.cfg_name == "vpc"
672
+ @vpc = self
673
+ end
674
+
675
+ # Google accounts usually have a useful default VPC we can use
676
+ if @vpc.nil? and @project_id and @cloud == "Google" and
677
+ self.class.can_live_in_vpc
678
+ MU.log "Seeing about default VPC for #{self}", MU::NOTICE
679
+ vpcs = MU::MommaCat.findStray(
680
+ "Google",
681
+ "vpc",
682
+ cloud_id: "default",
683
+ habitats: [@project_id],
684
+ credentials: @credentials,
685
+ dummy_ok: true,
686
+ debug: debug
687
+ )
688
+ @vpc = vpcs.first if !vpcs.nil? and vpcs.size > 0
689
+ end
690
+
691
+ # Special dependencies: LoadBalancers I've asked to attach to an
692
+ # instance.
693
+ if @config.has_key?("loadbalancers")
694
+ @loadbalancers = [] if !@loadbalancers
695
+ @config['loadbalancers'].each { |lb|
696
+ MU.log "Loading LoadBalancer for #{self}", MU::DEBUG, details: lb
697
+ if @dependencies.has_key?("loadbalancer") and
698
+ @dependencies["loadbalancer"].has_key?(lb['concurrent_load_balancer'])
699
+ @loadbalancers << @dependencies["loadbalancer"][lb['concurrent_load_balancer']]
700
+ else
701
+ if !lb.has_key?("existing_load_balancer") and
702
+ !lb.has_key?("deploy_id") and !@deploy.nil?
703
+ lb["deploy_id"] = @deploy.deploy_id
704
+ end
705
+ lbs = MU::MommaCat.findStray(
706
+ @config['cloud'],
707
+ "loadbalancer",
708
+ deploy_id: lb["deploy_id"],
709
+ cloud_id: lb['existing_load_balancer'],
710
+ name: lb['concurrent_load_balancer'],
711
+ region: @config["region"],
712
+ calling_deploy: @deploy,
713
+ dummy_ok: true
714
+ )
715
+ @loadbalancers << lbs.first if !lbs.nil? and lbs.size > 0
716
+ end
717
+ }
718
+ end
719
+
720
+ # Munge in external resources referenced by the existing_deploys
721
+ # keyword
722
+ if @config["existing_deploys"] && !@config["existing_deploys"].empty?
723
+ @config["existing_deploys"].each { |ext_deploy|
724
+ if ext_deploy["cloud_id"]
725
+ found = MU::MommaCat.findStray(
726
+ @config['cloud'],
727
+ ext_deploy["cloud_type"],
728
+ cloud_id: ext_deploy["cloud_id"],
729
+ region: @config['region'],
730
+ dummy_ok: false
731
+ ).first
732
+
733
+ MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
734
+ @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
735
+ elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
736
+ MU.log "#{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}"
737
+ found = MU::MommaCat.findStray(
738
+ @config['cloud'],
739
+ ext_deploy["cloud_type"],
740
+ deploy_id: ext_deploy["deploy_id"],
741
+ mu_name: ext_deploy["mu_name"],
742
+ region: @config['region'],
743
+ dummy_ok: false
744
+ ).first
745
+
746
+ MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
747
+ @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
748
+ else
749
+ MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
750
+ end
751
+ }
752
+ end
753
+
754
+ if @config['dns_records'] && !@config['dns_records'].empty?
755
+ @config['dns_records'].each { |dnsrec|
756
+ if dnsrec.has_key?("name")
757
+ if dnsrec['name'].start_with?(@deploy.deploy_id.downcase) && !dnsrec['name'].start_with?(@mu_name.downcase)
758
+ MU.log "DNS records for #{@mu_name} seem to be wrong, deleting from current config", MU::WARN, details: dnsrec
759
+ dnsrec.delete('name')
760
+ dnsrec.delete('target')
761
+ end
762
+ end
763
+ }
764
+ end
765
+
766
+ return [@dependencies, @vpc, @loadbalancers]
767
+ end
768
+
769
+ # Using the automatically-defined +@vpc+ from {dependencies} in
770
+ # conjunction with our config, return our configured subnets.
771
+ # @return [Array<MU::Cloud::VPC::Subnet>]
772
+ def mySubnets
773
+ dependencies
774
+ if !@vpc or !@config["vpc"]
775
+ return nil
776
+ end
777
+
778
+ if @config["vpc"]["subnet_id"] or @config["vpc"]["subnet_name"]
779
+ @config["vpc"]["subnets"] ||= []
780
+ subnet_block = {}
781
+ subnet_block["subnet_id"] = @config["vpc"]["subnet_id"] if @config["vpc"]["subnet_id"]
782
+ subnet_block["subnet_name"] = @config["vpc"]["subnet_name"] if @config["vpc"]["subnet_name"]
783
+ @config["vpc"]["subnets"] << subnet_block
784
+ @config["vpc"]["subnets"].uniq!
785
+ end
786
+
787
+ if (!@config["vpc"]["subnets"] or @config["vpc"]["subnets"].empty?) and
788
+ !@config["vpc"]["subnet_id"]
789
+ return @vpc.subnets
790
+ end
791
+
792
+ subnets = []
793
+ @config["vpc"]["subnets"].each { |subnet|
794
+ subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
795
+ raise MuError.new "Couldn't find a live subnet for #{self} matching #{subnet} in #{@vpc}", details: @vpc.subnets.map { |s| s.name }.join(",") if subnet_obj.nil?
796
+ subnets << subnet_obj
797
+ }
798
+
799
+ subnets
800
+ end
801
+
802
+ # @return [Array<MU::Cloud::FirewallRule>]
803
+ def myFirewallRules
804
+ dependencies
805
+
806
+ rules = []
807
+ if @dependencies.has_key?("firewall_rule")
808
+ rules = @dependencies['firewall_rule'].values
809
+ end
810
+ # XXX what other ways are these specified?
811
+
812
+ rules
813
+ end
814
+
815
+ # If applicable, allow this resource's NAT host blanket access via
816
+ # rules in its associated +admin+ firewall rule set.
817
+ def allowBastionAccess
818
+ return nil if !@nat or !@nat.is_a?(MU::Cloud::Server)
819
+
820
+ myFirewallRules.each { |acl|
821
+ if acl.config["admin"]
822
+ acl.addRule(@nat.listIPs, proto: "tcp")
823
+ acl.addRule(@nat.listIPs, proto: "udp")
824
+ acl.addRule(@nat.listIPs, proto: "icmp")
825
+ end
826
+ }
827
+ end
828
+
829
+ # A hook that is always called just before each instance method is
830
+ # invoked, so that we can ensure that repetitive setup tasks (like
831
+ # resolving +:resource_group+ for Azure resources) have always been
832
+ # done.
833
+ def resourceInitHook
834
+ @cloud ||= cloud
835
+ if @cloudparentclass.respond_to?(:resourceInitHook)
836
+ @cloudparentclass.resourceInitHook(@cloudobj, @deploy)
837
+ end
838
+ end
839
+
840
+ if File.exist?(MU.myRoot+"/modules/mu/cloud/#{cfg_name}.rb")
841
+ require "mu/cloud/#{cfg_name}"
842
+ end
843
+
844
+ # Wrap the instance methods that this cloud resource type has to
845
+ # implement.
846
+ MU::Cloud.resource_types[name.to_sym][:instance].each { |method|
847
+
848
+ define_method method do |*args|
849
+ return nil if @cloudobj.nil?
850
+ MU.log "Invoking #{@cloudobj}.#{method}", MU::DEBUG
851
+
852
+ # Go ahead and guarantee that we can't accidentally trigger these
853
+ # methods recursively.
854
+ @method_semaphore.synchronize {
855
+ # We're looking for recursion, not contention, so ignore some
856
+ # obviously harmless things.
857
+ if @method_locks.has_key?(method) and method != :findBastion and method != :cloud_id
858
+ MU.log "Double-call to cloud method #{method} for #{self}", MU::DEBUG, details: caller + ["competing call stack:"] + @method_locks[method]
859
+ end
860
+ @method_locks[method] = caller
861
+ }
862
+
863
+ # Make sure the describe() caches are fresh
864
+ @cloudobj.describe if method != :describe
865
+
866
+ # Don't run through dependencies on simple attr_reader lookups
867
+ if ![:dependencies, :cloud_id, :config, :mu_name].include?(method)
868
+ @cloudobj.dependencies
869
+ end
870
+
871
+ retval = nil
872
+ if !args.nil? and args.size == 1
873
+ retval = @cloudobj.method(method).call(args.first)
874
+ elsif !args.nil? and args.size > 0
875
+ retval = @cloudobj.method(method).call(*args)
876
+ else
877
+ retval = @cloudobj.method(method).call
878
+ end
879
+ if [:create, :groom, :postBoot, :toKitten].include?(method) and
880
+ (!@destroyed and !@cloudobj.destroyed)
881
+ deploydata = @cloudobj.method(:notify).call
882
+ @deploydata ||= deploydata # XXX I don't remember why we're not just doing this from the get-go; maybe because we prefer some mangling occurring in @deploy.notify?
883
+ if deploydata.nil? or !deploydata.is_a?(Hash)
884
+ MU.log "#{self} notify method did not return a Hash of deployment data, attempting to fill in with cloud descriptor #{@cloudobj.cloud_id}", MU::WARN
885
+ deploydata = MU.structToHash(@cloudobj.cloud_desc)
886
+ raise MuError, "Failed to collect metadata about #{self}" if deploydata.nil?
887
+ end
888
+ deploydata['cloud_id'] ||= @cloudobj.cloud_id if !@cloudobj.cloud_id.nil?
889
+ deploydata['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
890
+ deploydata['nodename'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
891
+ deploydata.delete("#MUOBJECT")
892
+ @deploy.notify(self.class.cfg_plural, @config['name'], deploydata, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
893
+ elsif method == :notify
894
+ if retval.nil?
895
+ MU.log self.to_s+" didn't return any metadata from notify", MU::WARN, details: @cloudobj.cloud_desc
896
+ end
897
+ retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
898
+ retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
899
+ @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
900
+ end
901
+ @method_semaphore.synchronize {
902
+ @method_locks.delete(method)
903
+ }
904
+
905
+ @deploydata = @cloudobj.deploydata
906
+ @config = @cloudobj.config
907
+ retval
908
+ end
909
+ } # end instance method list
910
+
911
+
912
+ } # end dynamic class generation block
913
+ } # end resource type iteration
914
+
915
+ require 'mu/cloud/winrm_sessions'
916
+ require 'mu/cloud/ssh_sessions'
917
+
918
+ end
919
+
920
+ end