cloud-mu 3.1.5 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  4. data/ansible/roles/mu-windows/files/config.xml +76 -0
  5. data/ansible/roles/mu-windows/tasks/main.yml +16 -0
  6. data/bin/mu-adopt +16 -12
  7. data/bin/mu-azure-tests +57 -0
  8. data/bin/mu-cleanup +2 -4
  9. data/bin/mu-configure +52 -0
  10. data/bin/mu-deploy +3 -3
  11. data/bin/mu-findstray-tests +25 -0
  12. data/bin/mu-gen-docs +2 -4
  13. data/bin/mu-load-config.rb +2 -1
  14. data/bin/mu-node-manage +15 -16
  15. data/bin/mu-run-tests +37 -12
  16. data/cloud-mu.gemspec +3 -3
  17. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  18. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  19. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  20. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  21. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  22. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  23. data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
  24. data/extras/clean-stock-amis +25 -19
  25. data/extras/generate-stock-images +1 -0
  26. data/extras/image-generators/AWS/win2k12.yaml +2 -0
  27. data/extras/image-generators/AWS/win2k16.yaml +2 -0
  28. data/extras/image-generators/AWS/win2k19.yaml +2 -0
  29. data/modules/mommacat.ru +1 -1
  30. data/modules/mu.rb +86 -98
  31. data/modules/mu/adoption.rb +373 -58
  32. data/modules/mu/cleanup.rb +214 -303
  33. data/modules/mu/cloud.rb +128 -1733
  34. data/modules/mu/cloud/database.rb +49 -0
  35. data/modules/mu/cloud/dnszone.rb +44 -0
  36. data/modules/mu/cloud/machine_images.rb +212 -0
  37. data/modules/mu/cloud/providers.rb +81 -0
  38. data/modules/mu/cloud/resource_base.rb +929 -0
  39. data/modules/mu/cloud/server.rb +40 -0
  40. data/modules/mu/cloud/server_pool.rb +1 -0
  41. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  42. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  43. data/modules/mu/cloud/wrappers.rb +169 -0
  44. data/modules/mu/config.rb +123 -81
  45. data/modules/mu/config/alarm.rb +2 -6
  46. data/modules/mu/config/bucket.rb +32 -3
  47. data/modules/mu/config/cache_cluster.rb +2 -2
  48. data/modules/mu/config/cdn.rb +100 -0
  49. data/modules/mu/config/collection.rb +1 -1
  50. data/modules/mu/config/container_cluster.rb +7 -2
  51. data/modules/mu/config/database.rb +84 -105
  52. data/modules/mu/config/database.yml +1 -2
  53. data/modules/mu/config/dnszone.rb +5 -4
  54. data/modules/mu/config/doc_helpers.rb +5 -6
  55. data/modules/mu/config/endpoint.rb +2 -1
  56. data/modules/mu/config/firewall_rule.rb +3 -19
  57. data/modules/mu/config/folder.rb +1 -1
  58. data/modules/mu/config/function.rb +17 -8
  59. data/modules/mu/config/group.rb +1 -1
  60. data/modules/mu/config/habitat.rb +1 -1
  61. data/modules/mu/config/job.rb +89 -0
  62. data/modules/mu/config/loadbalancer.rb +57 -11
  63. data/modules/mu/config/log.rb +1 -1
  64. data/modules/mu/config/msg_queue.rb +1 -1
  65. data/modules/mu/config/nosqldb.rb +1 -1
  66. data/modules/mu/config/notifier.rb +8 -19
  67. data/modules/mu/config/ref.rb +92 -14
  68. data/modules/mu/config/role.rb +1 -1
  69. data/modules/mu/config/schema_helpers.rb +38 -37
  70. data/modules/mu/config/search_domain.rb +1 -1
  71. data/modules/mu/config/server.rb +12 -13
  72. data/modules/mu/config/server_pool.rb +3 -7
  73. data/modules/mu/config/storage_pool.rb +1 -1
  74. data/modules/mu/config/tail.rb +11 -0
  75. data/modules/mu/config/user.rb +1 -1
  76. data/modules/mu/config/vpc.rb +27 -23
  77. data/modules/mu/config/vpc.yml +0 -1
  78. data/modules/mu/defaults/AWS.yaml +90 -90
  79. data/modules/mu/defaults/Azure.yaml +1 -0
  80. data/modules/mu/defaults/Google.yaml +1 -0
  81. data/modules/mu/deploy.rb +34 -20
  82. data/modules/mu/groomer.rb +16 -1
  83. data/modules/mu/groomers/ansible.rb +69 -4
  84. data/modules/mu/groomers/chef.rb +51 -4
  85. data/modules/mu/logger.rb +120 -144
  86. data/modules/mu/master.rb +97 -4
  87. data/modules/mu/mommacat.rb +160 -874
  88. data/modules/mu/mommacat/daemon.rb +23 -14
  89. data/modules/mu/mommacat/naming.rb +110 -3
  90. data/modules/mu/mommacat/search.rb +497 -0
  91. data/modules/mu/mommacat/storage.rb +252 -194
  92. data/modules/mu/{clouds → providers}/README.md +1 -1
  93. data/modules/mu/{clouds → providers}/aws.rb +258 -57
  94. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  95. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  96. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  97. data/modules/mu/providers/aws/cdn.rb +782 -0
  98. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  99. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
  100. data/modules/mu/providers/aws/database.rb +1744 -0
  101. data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
  102. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  103. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
  104. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  105. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  106. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  107. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  108. data/modules/mu/providers/aws/job.rb +466 -0
  109. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
  110. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  111. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  112. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  113. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  114. data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
  115. data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
  116. data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
  117. data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
  118. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  119. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  120. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  121. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  122. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  123. data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
  124. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  125. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  126. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  127. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  128. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  129. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  130. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  131. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  132. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  133. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  134. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  135. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  136. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  137. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  138. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  139. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  140. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  141. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  142. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  143. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  144. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  145. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  146. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  147. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  148. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  149. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  150. data/modules/mu/{clouds → providers}/google.rb +29 -6
  151. data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
  152. data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
  153. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  154. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  155. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  156. data/modules/mu/{clouds → providers}/google/function.rb +6 -6
  157. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  158. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  159. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  160. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  161. data/modules/mu/{clouds → providers}/google/server.rb +41 -24
  162. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  163. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  164. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  165. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  166. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  167. data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
  168. data/modules/tests/aws-jobs-functions.yaml +46 -0
  169. data/modules/tests/centos6.yaml +15 -0
  170. data/modules/tests/centos7.yaml +15 -0
  171. data/modules/tests/centos8.yaml +12 -0
  172. data/modules/tests/ecs.yaml +2 -2
  173. data/modules/tests/eks.yaml +1 -1
  174. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  175. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  176. data/modules/tests/microservice_app.yaml +288 -0
  177. data/modules/tests/rds.yaml +108 -0
  178. data/modules/tests/regrooms/rds.yaml +123 -0
  179. data/modules/tests/server-with-scrub-muisms.yaml +1 -1
  180. data/modules/tests/super_complex_bok.yml +2 -2
  181. data/modules/tests/super_simple_bok.yml +3 -5
  182. data/spec/mu/clouds/azure_spec.rb +2 -2
  183. metadata +122 -92
  184. data/modules/mu/clouds/aws/database.rb +0 -1974
  185. data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -0,0 +1,49 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic methods for all Database implementations
21
+ class Database
22
+
23
+ # Getting the password for a database's master user, and saving it in a database / cluster specific vault
24
+ def getPassword
25
+ if @config['password'].nil?
26
+ if @config['auth_vault'] && !@config['auth_vault'].empty?
27
+ @config['password'] = @groomclass.getSecret(
28
+ vault: @config['auth_vault']['vault'],
29
+ item: @config['auth_vault']['item'],
30
+ field: @config['auth_vault']['password_field']
31
+ )
32
+ else
33
+ # Should we use random instead?
34
+ @config['password'] = Password.pronounceable(10..12)
35
+ end
36
+ end
37
+
38
+ creds = {
39
+ "username" => @config["master_user"],
40
+ "password" => @config["password"]
41
+ }
42
+ @groomclass.saveSecret(vault: @mu_name, item: "database_credentials", data: creds)
43
+ end
44
+
45
+ end
46
+
47
+ end
48
+
49
+ end
@@ -0,0 +1,44 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic methods for all DNSZone implementations
21
+ class DNSZone
22
+
23
+ # Set a generic .platform-mu DNS entry for a resource, and return the name
24
+ # that was set.
25
+ def self.genericMuDNSEntry(*flags)
26
+ # XXX have this switch on a global config for where Mu puts its DNS
27
+ MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone").genericMuDNSEntry(flags.first)
28
+ end
29
+
30
+ # Wrapper for {MU::Cloud::AWS::DNSZone.manageRecord}. Spawns threads to create all
31
+ # requested records in background and returns immediately.
32
+ def self.createRecordsFromConfig(*flags)
33
+ cloudclass = MU::Cloud.resourceClass(MU::Config.defaultCloud, "DNSZone")
34
+ if !flags.nil? and flags.size == 1
35
+ cloudclass.createRecordsFromConfig(flags.first)
36
+ else
37
+ cloudclass.createRecordsFromConfig(*flags)
38
+ end
39
+ end
40
+ end
41
+
42
+ end
43
+
44
+ end
@@ -0,0 +1,212 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # The public AWS S3 bucket where we expect to find YAML files listing our
21
+ # standard base images for various platforms.
22
+ BASE_IMAGE_BUCKET = "cloudamatic"
23
+ # The path in the AWS S3 bucket where we expect to find YAML files listing
24
+ # our standard base images for various platforms.
25
+ BASE_IMAGE_PATH = "/images"
26
+
27
+ # Aliases for platform names, in case we don't have actual images built for
28
+ # them.
29
+ PLATFORM_ALIASES = {
30
+ "linux" => "centos7",
31
+ "windows" => "win2k12r2",
32
+ "win2k12" => "win2k12r2",
33
+ "ubuntu" => "ubuntu16",
34
+ "centos" => "centos7",
35
+ "rhel7" => "rhel71",
36
+ "rhel" => "rhel71",
37
+ "amazon" => "amazon2016"
38
+ }
39
+
40
+ @@image_fetch_cache = {}
41
+ @@platform_cache = []
42
+ @@image_fetch_semaphore = Mutex.new
43
+
44
+ # Rifle our image lists from {MU::Cloud.getStockImage} and return a list
45
+ # of valid +platform+ names.
46
+ # @return [Array<String>]
47
+ def self.listPlatforms
48
+ return @@platform_cache if @@platform_cache and !@@platform_cache.empty?
49
+ @@platform_cache = MU::Cloud.supportedClouds.map { |cloud|
50
+ begin
51
+ resourceClass(cloud, :Server)
52
+ rescue MU::Cloud::MuCloudResourceNotImplemented, MU::MuError
53
+ next
54
+ end
55
+
56
+ images = MU::Cloud.getStockImage(cloud, quiet: true)
57
+ if images
58
+ images.keys
59
+ else
60
+ nil
61
+ end
62
+ }.flatten.uniq
63
+ @@platform_cache.delete(nil)
64
+ @@platform_cache.sort
65
+ @@platform_cache
66
+ end
67
+
68
+ # Locate a base image for a {MU::Cloud::Server} resource. First we check
69
+ # Mu's public bucket, which should list the latest and greatest. If we can't
70
+ # fetch that, then we fall back to a YAML file that's bundled as part of Mu,
71
+ # but which will typically be less up-to-date.
72
+ # @param cloud [String]: The cloud provider for which to return an image list
73
+ # @param platform [String]: The supported platform for which to return an image or images. If not specified, we'll return our entire library for the appropriate cloud provider.
74
+ # @param region [String]: The region for which the returned image or images should be supported, for cloud providers which require it (such as AWS).
75
+ # @param fail_hard [Boolean]: Raise an exception on most errors, such as an inability to reach our public listing, lack of matching images, etc.
76
+ # @return [Hash,String,nil]
77
+ def self.getStockImage(cloud = MU::Config.defaultCloud, platform: nil, region: nil, fail_hard: false, quiet: false)
78
+
79
+ if !MU::Cloud.supportedClouds.include?(cloud)
80
+ MU.log "'#{cloud}' is not a supported cloud provider! Available providers:", MU::ERR, details: MU::Cloud.supportedClouds
81
+ raise MuError, "'#{cloud}' is not a supported cloud provider!"
82
+ end
83
+
84
+ urls = ["http://"+BASE_IMAGE_BUCKET+".s3-website-us-east-1.amazonaws.com"+BASE_IMAGE_PATH]
85
+ if $MU_CFG and $MU_CFG['custom_images_url']
86
+ urls << $MU_CFG['custom_images_url']
87
+ end
88
+
89
+ images = nil
90
+ urls.each { |base_url|
91
+ @@image_fetch_semaphore.synchronize {
92
+ if @@image_fetch_cache[cloud] and (Time.now - @@image_fetch_cache[cloud]['time']) < 30
93
+ images = @@image_fetch_cache[cloud]['contents'].dup
94
+ else
95
+ begin
96
+ Timeout.timeout(2) do
97
+ response = open("#{base_url}/#{cloud}.yaml").read
98
+ images ||= {}
99
+ images.deep_merge!(YAML.load(response))
100
+ break
101
+ end
102
+ rescue StandardError => e
103
+ if fail_hard
104
+ raise MuError, "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})"
105
+ else
106
+ MU.log "Failed to fetch stock images from #{base_url}/#{cloud}.yaml (#{e.message})", MU::WARN if !quiet
107
+ end
108
+ end
109
+ end
110
+ }
111
+ }
112
+
113
+ @@image_fetch_semaphore.synchronize {
114
+ @@image_fetch_cache[cloud] = {
115
+ 'contents' => images.dup,
116
+ 'time' => Time.now
117
+ }
118
+ }
119
+
120
+ backwards_compat = {
121
+ "AWS" => "amazon_images",
122
+ "Google" => "google_images",
123
+ }
124
+
125
+ # Load from inside our repository, if we didn't get images elsewise
126
+ if images.nil?
127
+ [backwards_compat[cloud], cloud].each { |file|
128
+ next if file.nil?
129
+ if File.exist?("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml")
130
+ images = YAML.load(File.read("#{MU.myRoot}/modules/mu/defaults/#{file}.yaml"))
131
+ break
132
+ end
133
+ }
134
+ end
135
+
136
+ # Now overlay local overrides, both of the systemwide (/opt/mu/etc) and
137
+ # per-user (~/.mu/etc) variety.
138
+ [backwards_compat[cloud], cloud].each { |file|
139
+ next if file.nil?
140
+ if File.exist?("#{MU.etcDir}/#{file}.yaml")
141
+ images ||= {}
142
+ images.deep_merge!(YAML.load(File.read("#{MU.etcDir}/#{file}.yaml")))
143
+ end
144
+ if Process.uid != 0
145
+ basepath = Etc.getpwuid(Process.uid).dir+"/.mu/etc"
146
+ if File.exist?("#{basepath}/#{file}.yaml")
147
+ images ||= {}
148
+ images.deep_merge!(YAML.load(File.read("#{basepath}/#{file}.yaml")))
149
+ end
150
+ end
151
+ }
152
+
153
+ if images.nil?
154
+ if fail_hard
155
+ raise MuError, "Failed to find any base images for #{cloud}"
156
+ else
157
+ MU.log "Failed to find any base images for #{cloud}", MU::WARN if !quiet
158
+ return nil
159
+ end
160
+ end
161
+
162
+ PLATFORM_ALIASES.each_pair { |a, t|
163
+ if images[t] and !images[a]
164
+ images[a] = images[t]
165
+ end
166
+ }
167
+
168
+ if platform
169
+ if !images[platform]
170
+ if fail_hard
171
+ raise MuError, "No base image for platform #{platform} in cloud #{cloud}"
172
+ else
173
+ MU.log "No base image for platform #{platform} in cloud #{cloud}", MU::WARN if !quiet
174
+ return nil
175
+ end
176
+ end
177
+ images = images[platform]
178
+
179
+ if region
180
+ # We won't fuss about the region argument if this isn't a cloud that
181
+ # has regions, just quietly don't bother.
182
+ if images.is_a?(Hash)
183
+ if images[region]
184
+ images = images[region]
185
+ else
186
+ if fail_hard
187
+ raise MuError, "No base image for platform #{platform} in cloud #{cloud} region #{region} found"
188
+ else
189
+ MU.log "No base image for platform #{platform} in cloud #{cloud} region #{region} found", MU::WARN if !quiet
190
+ return nil
191
+ end
192
+ end
193
+ end
194
+ end
195
+ else
196
+ if region
197
+ images.values.each { |regions|
198
+ # Filter to match our requested region, but for all the platforms,
199
+ # since we didn't specify one.
200
+ if regions.is_a?(Hash)
201
+ regions.delete_if { |r| r != region }
202
+ end
203
+ }
204
+ end
205
+ end
206
+
207
+ images
208
+ end
209
+
210
+ end
211
+
212
+ end
@@ -0,0 +1,81 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # List of known/supported Cloud providers. This may be modified at runtime
21
+ # if an implemention is defective or missing required methods.
22
+ @@supportedCloudList = ['AWS', 'CloudFormation', 'Google', 'Azure']
23
+
24
+ # List of known/supported Cloud providers
25
+ # @return [Array<String>]
26
+ def self.supportedClouds
27
+ @@supportedCloudList
28
+ end
29
+
30
+ # Raise an exception if the cloud provider specified isn't valid
31
+ def self.cloudClass(cloud)
32
+ if cloud.nil? or !supportedClouds.include?(cloud.to_s)
33
+ raise MuError, "Cloud provider #{cloud} is not supported"
34
+ end
35
+ Object.const_get("MU").const_get("Cloud").const_get(cloud.to_s)
36
+ end
37
+
38
+ # List of known/supported Cloud providers for which we have at least one
39
+ # set of credentials configured.
40
+ # @return [Array<String>]
41
+ def self.availableClouds
42
+ available = []
43
+ MU::Cloud.supportedClouds.each { |cloud|
44
+ begin
45
+ cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
46
+ next if cloudbase.listCredentials.nil? or cloudbase.listCredentials.empty?
47
+ available << cloud
48
+ rescue NameError
49
+ end
50
+ }
51
+
52
+ available
53
+ end
54
+
55
+ # Raise an exception if the cloud provider specified isn't valid or we
56
+ # don't have any credentials configured for it.
57
+ def self.assertAvailableCloud(cloud)
58
+ if cloud.nil? or availableClouds.include?(cloud.to_s)
59
+ raise MuError, "Cloud provider #{cloud} is not available"
60
+ end
61
+ end
62
+
63
+ # Load the container class for each cloud we know about, and inject autoload
64
+ # code for each of its supported resource type classes.
65
+ failed = []
66
+ MU::Cloud.supportedClouds.each { |cloud|
67
+ require "mu/providers/#{cloud.downcase}"
68
+ cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
69
+ @@generic_class_methods_toplevel.each { |method|
70
+ if !cloudclass.respond_to?(method)
71
+ MU.log "MU::Cloud::#{cloud} has not implemented required class method #{method}, disabling", MU::ERR
72
+ failed << cloud
73
+ end
74
+ }
75
+ }
76
+ failed.uniq!
77
+ @@supportedCloudList = @@supportedCloudList - failed
78
+
79
+ end
80
+
81
+ end
@@ -0,0 +1,929 @@
1
+ # Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ # Plugins under this namespace serve as interfaces to cloud providers and
17
+ # other provisioning layers.
18
+ class Cloud
19
+
20
+ # Generic class methods (.find, .cleanup, etc) are defined in wrappers.rb
21
+ require 'mu/cloud/wrappers'
22
+
23
+ @@resource_types.each_key { |name|
24
+ Object.const_get("MU").const_get("Cloud").const_get(name).class_eval {
25
+ attr_reader :cloudclass
26
+ attr_reader :cloudobj
27
+ attr_reader :destroyed
28
+ attr_reader :delayed_save
29
+
30
+ # Print something palatable when we're called in a string context.
31
+ def to_s
32
+ fullname = "#{self.class.shortname}"
33
+ if !@cloudobj.nil? and !@cloudobj.mu_name.nil?
34
+ @mu_name ||= @cloudobj.mu_name
35
+ end
36
+ if !@mu_name.nil? and !@mu_name.empty?
37
+ fullname = fullname + " '#{@mu_name}'"
38
+ end
39
+ if !@cloud_id.nil?
40
+ fullname = fullname + " (#{@cloud_id})"
41
+ end
42
+ return fullname
43
+ end
44
+
45
+ # Set our +deploy+ and +deploy_id+ attributes, optionally doing so even
46
+ # if they have already been set.
47
+ #
48
+ # @param mommacat [MU::MommaCat]: The deploy to which we're being told we belong
49
+ # @param force [Boolean]: Set even if we already have a deploy object
50
+ # @return [String]: Our new +deploy_id+
51
+ def intoDeploy(mommacat, force: false)
52
+ if force or (!@deploy)
53
+ MU.log "Inserting #{self} [#{self.object_id}] into #{mommacat.deploy_id} as a #{@config['name']}", MU::DEBUG
54
+
55
+ @deploy = mommacat
56
+ @deploy.addKitten(@cloudclass.cfg_plural, @config['name'], self)
57
+ @deploy_id = @deploy.deploy_id
58
+ @cloudobj.intoDeploy(mommacat, force: force) if @cloudobj
59
+ end
60
+ @deploy_id
61
+ end
62
+
63
+ # Return the +virtual_name+ config field, if it is set.
64
+ # @param name [String]: If set, will only return a value if +virtual_name+ matches this string
65
+ # @return [String,nil]
66
+ def virtual_name(name = nil)
67
+ if @config and @config['virtual_name'] and
68
+ (!name or name == @config['virtual_name'])
69
+ return @config['virtual_name']
70
+ end
71
+ nil
72
+ end
73
+
74
+ # @param mommacat [MU::MommaCat]: The deployment containing this cloud resource
75
+ # @param mu_name [String]: Optional- specify the full Mu resource name of an existing resource to load, instead of creating a new one
76
+ # @param cloud_id [String]: Optional- specify the cloud provider's identifier for an existing resource to load, instead of creating a new one
77
+ # @param kitten_cfg [Hash]: The parse configuration for this object from {MU::Config}
78
+ def initialize(**args)
79
+ raise MuError, "Cannot invoke Cloud objects without a configuration" if args[:kitten_cfg].nil?
80
+
81
+ # We are a parent wrapper object. Initialize our child object and
82
+ # housekeeping bits accordingly.
83
+ if self.class.name =~ /^MU::Cloud::([^:]+)$/
84
+ @live = true
85
+ @delayed_save = args[:delayed_save]
86
+ @method_semaphore = Mutex.new
87
+ @method_locks = {}
88
+ if args[:mommacat]
89
+ MU.log "Initializing an instance of #{self.class.name} in #{args[:mommacat].deploy_id} #{mu_name}", MU::DEBUG, details: args[:kitten_cfg]
90
+ elsif args[:mu_name].nil?
91
+ raise MuError, "Can't instantiate a MU::Cloud object with a live deploy or giving us a mu_name"
92
+ else
93
+ MU.log "Initializing a detached #{self.class.name} named #{args[:mu_name]}", MU::DEBUG, details: args[:kitten_cfg]
94
+ end
95
+
96
+ my_cloud = args[:kitten_cfg]['cloud'].to_s || MU::Config.defaultCloud
97
+ if (my_cloud.nil? or my_cloud.empty?) and args[:mommacat]
98
+ my_cloud = args[:mommacat].original_config['cloud']
99
+ end
100
+ if my_cloud.nil? or !MU::Cloud.supportedClouds.include?(my_cloud)
101
+ raise MuError, "Can't instantiate a MU::Cloud object without a valid cloud (saw '#{my_cloud}')"
102
+ end
103
+ @cloudclass = MU::Cloud.resourceClass(my_cloud, self.class.shortname)
104
+ @cloud_desc_cache ||= args[:from_cloud_desc] if args[:from_cloud_desc]
105
+ @cloudparentclass = MU::Cloud.cloudClass(my_cloud)
106
+ @cloudobj = @cloudclass.new(
107
+ mommacat: args[:mommacat],
108
+ kitten_cfg: args[:kitten_cfg],
109
+ cloud_id: args[:cloud_id],
110
+ mu_name: args[:mu_name]
111
+ )
112
+ raise MuError, "Unknown error instantiating #{self}" if @cloudobj.nil?
113
+ # These should actually call the method live instead of caching a static value
114
+ PUBLIC_ATTRS.each { |a|
115
+ begin
116
+ instance_variable_set(("@"+a.to_s).to_sym, @cloudobj.send(a))
117
+ rescue NoMethodError => e
118
+ MU.log "#{@cloudclass.name} failed to implement method '#{a}'", MU::ERR, details: e.message
119
+ raise e
120
+ end
121
+ }
122
+ @deploy ||= args[:mommacat]
123
+ @deploy_id ||= @deploy.deploy_id if @deploy
124
+
125
+ # Register with the containing deployment
126
+ if !@deploy.nil? and !@cloudobj.mu_name.nil? and
127
+ !@cloudobj.mu_name.empty? and !args[:delay_descriptor_load]
128
+ describe # XXX is this actually safe here?
129
+ @deploy.addKitten(self.class.cfg_name, @config['name'], self)
130
+ elsif !@deploy.nil? and @cloudobj.mu_name.nil?
131
+ MU.log "#{self} in #{@deploy.deploy_id} didn't generate a mu_name after being loaded/initialized, dependencies on this resource will probably be confused!", MU::ERR, details: [caller, args.keys]
132
+ end
133
+
134
+ # We are actually a child object invoking this via super() from its
135
+ # own initialize(), so initialize all the attributes and instance
136
+ # variables we know to be universal.
137
+ else
138
+ class << self
139
+ # Declare attributes that everyone should have
140
+ PUBLIC_ATTRS.each { |a|
141
+ attr_reader a
142
+ }
143
+ end
144
+ # XXX this butchers ::Id and ::Ref objects that might be used by dependencies() to good effect, but we also can't expect our implementations to cope with knowing when a .to_s has to be appended to things at random
145
+ @config = MU::Config.manxify(args[:kitten_cfg]) || MU::Config.manxify(args[:config])
146
+
147
+ if !@config
148
+ MU.log "Missing config arguments in setInstanceVariables, can't initialize a cloud object without it", MU::ERR, details: args.keys
149
+ raise MuError, "Missing config arguments in setInstanceVariables"
150
+ end
151
+
152
+ @deploy = args[:mommacat] || args[:deploy]
153
+ @cloud_desc_cache ||= args[:from_cloud_desc] if args[:from_cloud_desc]
154
+
155
+ @credentials = args[:credentials]
156
+ @credentials ||= @config['credentials']
157
+
158
+ @cloud = @config['cloud']
159
+ if !@cloud
160
+ if self.class.name =~ /^MU::Cloud::([^:]+)(?:::.+|$)/
161
+ cloudclass_name = Regexp.last_match[1]
162
+ if MU::Cloud.supportedClouds.include?(cloudclass_name)
163
+ @cloud = cloudclass_name
164
+ end
165
+ end
166
+ end
167
+ if !@cloud
168
+ raise MuError, "Failed to determine what cloud #{self} should be in!"
169
+ end
170
+
171
+ @environment = @config['environment']
172
+ if @deploy
173
+ @deploy_id = @deploy.deploy_id
174
+ @appname = @deploy.appname
175
+ end
176
+
177
+ @cloudclass = MU::Cloud.resourceClass(@cloud, self.class.shortname)
178
+ @cloudparentclass = MU::Cloud.cloudClass(@cloud)
179
+
180
+ # A pre-existing object, you say?
181
+ if args[:cloud_id]
182
+
183
+ # TODO implement ::Id for every cloud... and they should know how to get from
184
+ # cloud_desc to a fully-resolved ::Id object, not just the short string
185
+
186
+ @cloud_id = args[:cloud_id]
187
+ describe(cloud_id: @cloud_id)
188
+ @habitat_id = habitat_id # effectively, cache this
189
+
190
+ # If we can build us an ::Id object for @cloud_id instead of a
191
+ # string, do so.
192
+ begin
193
+ idclass = @cloudparentclass.const_get(:Id)
194
+ long_id = if @deploydata and @deploydata[idclass.idattr.to_s]
195
+ @deploydata[idclass.idattr.to_s]
196
+ elsif self.respond_to?(idclass.idattr)
197
+ self.send(idclass.idattr)
198
+ end
199
+
200
+ @cloud_id = idclass.new(long_id) if !long_id.nil? and !long_id.empty?
201
+ # 1 see if we have the value on the object directly or in deploy data
202
+ # 2 set an attr_reader with the value
203
+ # 3 rewrite our @cloud_id attribute with a ::Id object
204
+ rescue NameError, MU::Cloud::MuCloudResourceNotImplemented
205
+ end
206
+
207
+ end
208
+
209
+ # Use pre-existing mu_name (we're probably loading an extant deploy)
210
+ # if available
211
+ if args[:mu_name]
212
+ @mu_name = args[:mu_name].dup
213
+ # If scrub_mu_isms is set, our mu_name is always just the bare name
214
+ # field of the resource.
215
+ elsif @config['scrub_mu_isms']
216
+ @mu_name = @config['name'].dup
217
+ # XXX feck it insert an inheritable method right here? Set a default? How should resource implementations determine whether they're instantiating a new object?
218
+ end
219
+
220
+ @tags = {}
221
+ if !@config['scrub_mu_isms']
222
+ @tags = @deploy ? @deploy.listStandardTags : MU::MommaCat.listStandardTags
223
+ end
224
+ if @config['tags']
225
+ @config['tags'].each { |tag|
226
+ @tags[tag['key']] = tag['value']
227
+ }
228
+ end
229
+
230
+ MU::MommaCat.listOptionalTags.each_pair { |k, v|
231
+ @tags[k] ||= v if v
232
+ }
233
+
234
+ if @cloudparentclass.respond_to?(:resourceInitHook)
235
+ @cloudparentclass.resourceInitHook(self, @deploy)
236
+ end
237
+
238
+ # Add cloud-specific instance methods for our resource objects to
239
+ # inherit.
240
+ if @cloudparentclass.const_defined?(:AdditionalResourceMethods)
241
+ self.extend @cloudparentclass.const_get(:AdditionalResourceMethods)
242
+ end
243
+
244
+ if ["Server", "ServerPool"].include?(self.class.shortname) and @deploy
245
+ @mu_name ||= @deploy.getResourceName(@config['name'], need_unique_string: @config.has_key?("basis"))
246
+ if self.class.shortname == "Server"
247
+ @groomer = MU::Groomer.new(self)
248
+ end
249
+
250
+ @groomclass = MU::Groomer.loadGroomer(@config["groomer"])
251
+
252
+ if windows? or @config['active_directory'] and !@mu_windows_name
253
+ if !@deploydata.nil? and !@deploydata['mu_windows_name'].nil?
254
+ @mu_windows_name = @deploydata['mu_windows_name']
255
+ else
256
+ # Use the same random differentiator as the "real" name if we're
257
+ # from a ServerPool. Helpful for admin sanity.
258
+ unq = @mu_name.sub(/^.*?-(...)$/, '\1')
259
+ if @config['basis'] and !unq.nil? and !unq.empty?
260
+ @mu_windows_name = @deploy.getResourceName(@config['name'], max_length: 15, need_unique_string: true, use_unique_string: unq, reuse_unique_string: true)
261
+ else
262
+ @mu_windows_name = @deploy.getResourceName(@config['name'], max_length: 15, need_unique_string: true)
263
+ end
264
+ end
265
+ end
266
+ class << self
267
+ attr_reader :groomer
268
+ attr_reader :groomerclass
269
+ attr_accessor :mu_windows_name # XXX might be ok as reader now
270
+ end
271
+ end
272
+ @tags["Name"] ||= @mu_name if @mu_name
273
+ end
274
+
275
+ end
276
+
277
+ def cloud
278
+ if @cloud
279
+ @cloud
280
+ elsif @config and @config['cloud']
281
+ @config['cloud']
282
+ elsif self.class.name =~ /^MU::Cloud::([^:]+)::.+/
283
+ cloudclass_name = Regexp.last_match[1]
284
+ if MU::Cloud.supportedClouds.include?(cloudclass_name)
285
+ cloudclass_name
286
+ else
287
+ nil
288
+ end
289
+ else
290
+ nil
291
+ end
292
+ end
293
+
294
+
295
+ # Remove all metadata and cloud resources associated with this object
296
+ def destroy
297
+ if self.class.cfg_name == "server"
298
+ begin
299
+ ip = canonicalIP
300
+ MU::Master.removeIPFromSSHKnownHosts(ip) if ip
301
+ if @deploy and @deploy.deployment and
302
+ @deploy.deployment['servers'] and @config['name']
303
+ me = @deploy.deployment['servers'][@config['name']][@mu_name]
304
+ if me
305
+ ["private_ip_address", "public_ip_address"].each { |field|
306
+ if me[field]
307
+ MU::Master.removeIPFromSSHKnownHosts(me[field])
308
+ end
309
+ }
310
+ if me["private_ip_list"]
311
+ me["private_ip_list"].each { |private_ip|
312
+ MU::Master.removeIPFromSSHKnownHosts(private_ip)
313
+ }
314
+ end
315
+ end
316
+ end
317
+ rescue MU::MuError => e
318
+ MU.log e.message, MU::WARN
319
+ end
320
+ end
321
+ if !@cloudobj.nil? and !@cloudobj.groomer.nil?
322
+ @cloudobj.groomer.cleanup
323
+ elsif !@groomer.nil?
324
+ @groomer.cleanup
325
+ end
326
+ if !@deploy.nil?
327
+ if !@cloudobj.nil? and !@config.nil? and !@cloudobj.mu_name.nil?
328
+ @deploy.notify(self.class.cfg_plural, @config['name'], nil, mu_name: @cloudobj.mu_name, remove: true, triggering_node: @cloudobj, delayed_save: @delayed_save)
329
+ elsif !@mu_name.nil?
330
+ @deploy.notify(self.class.cfg_plural, @config['name'], nil, mu_name: @mu_name, remove: true, triggering_node: self, delayed_save: @delayed_save)
331
+ end
332
+ @deploy.removeKitten(self)
333
+ end
334
+ # Make sure that if notify gets called again it won't go returning a
335
+ # bunch of now-bogus metadata.
336
+ @destroyed = true
337
+ if !@cloudobj.nil?
338
+ def @cloudobj.notify
339
+ {}
340
+ end
341
+ else
342
+ def notify
343
+ {}
344
+ end
345
+ end
346
+ end
347
+
348
+ # Return the cloud object's idea of where it lives (project, account,
349
+ # etc) in the form of an identifier. If not applicable for this object,
350
+ # we expect to return +nil+.
351
+ # @return [String,nil]
352
+ def habitat(nolookup: true)
353
+ return nil if ["folder", "habitat"].include?(self.class.cfg_name)
354
+ if @cloudobj
355
+ @cloudparentclass.habitat(@cloudobj, nolookup: nolookup, deploy: @deploy)
356
+ else
357
+ @cloudparentclass.habitat(self, nolookup: nolookup, deploy: @deploy)
358
+ end
359
+ end
360
+
361
+ def habitat_id(nolookup: false)
362
+ @habitat_id ||= habitat(nolookup: nolookup)
363
+ @habitat_id
364
+ end
365
+
366
+ # We're fundamentally a wrapper class, so go ahead and reroute requests
367
+ # that are meant for our wrapped object.
368
+ def method_missing(method_sym, *arguments)
369
+ if @cloudobj
370
+ MU.log "INVOKING #{method_sym} FROM PARENT CLOUD OBJECT #{self}", MU::DEBUG, details: arguments
371
+ @cloudobj.method(method_sym).call(*arguments)
372
+ else
373
+ raise NoMethodError, "No such instance method #{method_sym} available on #{self.class.name}"
374
+ end
375
+ end
376
+
377
+ # Merge the passed hash into the existing configuration hash of this
378
+ # cloud object. Currently this is only used by the {MU::Adoption}
379
+ # module. I don't love exposing this to the whole internal API, but I'm
380
+ # probably overthinking that.
381
+ # @param newcfg [Hash]
382
+ def config!(newcfg)
383
+ @config.merge!(newcfg)
384
+ end
385
+
386
+ def cloud_desc(use_cache: true)
387
+ describe
388
+
389
+ if !@cloudobj.nil?
390
+ if @cloudobj.class.instance_methods(false).include?(:cloud_desc)
391
+ @cloud_desc_cache ||= @cloudobj.cloud_desc
392
+ end
393
+ end
394
+ if !@config.nil? and !@cloud_id.nil? and (!use_cache or @cloud_desc_cache.nil?)
395
+ # The find() method should be returning a Hash with the cloud_id
396
+ # as a key and a cloud platform descriptor as the value.
397
+ begin
398
+ args = {
399
+ :region => @config['region'],
400
+ :cloud => @config['cloud'],
401
+ :cloud_id => @cloud_id,
402
+ :credentials => @credentials,
403
+ :project => habitat_id, # XXX this belongs in our required_instance_methods hack
404
+ :flags => @config
405
+ }
406
+ @cloudparentclass.required_instance_methods.each { |m|
407
+ # if respond_to?(m)
408
+ # args[m] = method(m).call
409
+ # else
410
+ args[m] = instance_variable_get(("@"+m.to_s).to_sym)
411
+ # end
412
+ }
413
+
414
+ matches = self.class.find(args)
415
+ if !matches.nil? and matches.is_a?(Hash)
416
+ # XXX or if the hash is keyed with an ::Id element, oh boy
417
+ # puts matches[@cloud_id][:self_link]
418
+ # puts matches[@cloud_id][:url]
419
+ # if matches[@cloud_id][:self_link]
420
+ # @url ||= matches[@cloud_id][:self_link]
421
+ # elsif matches[@cloud_id][:url]
422
+ # @url ||= matches[@cloud_id][:url]
423
+ # elsif matches[@cloud_id][:arn]
424
+ # @arn ||= matches[@cloud_id][:arn]
425
+ # end
426
+ if matches[@cloud_id]
427
+ @cloud_desc_cache = matches[@cloud_id]
428
+ else
429
+ matches.each_pair { |k, v| # flatten out ::Id objects just in case
430
+ if @cloud_id.to_s == k.to_s
431
+ @cloud_desc_cache = v
432
+ break
433
+ end
434
+ }
435
+ end
436
+ end
437
+
438
+ if !@cloud_desc_cache
439
+ MU.log "cloud_desc via #{self.class.name}.find() failed to locate a live object.\nWas called by #{caller(1..1)}", MU::WARN, details: args
440
+ end
441
+ rescue StandardError => e
442
+ MU.log "Got #{e.inspect} trying to find cloud handle for #{self.class.shortname} #{@mu_name} (#{@cloud_id})", MU::WARN
443
+ raise e
444
+ end
445
+ end
446
+
447
+ return @cloud_desc_cache
448
+ end
449
+
450
+ # Retrieve all of the known metadata for this resource.
451
+ # @param cloud_id [String]: The cloud platform's identifier for the resource we're describing. Makes lookups more efficient.
452
+ # @return [Array<Hash>]: mu_name, config, deploydata
453
+ def describe(cloud_id: nil)
454
+ if cloud_id.nil? and !@cloudobj.nil?
455
+ @cloud_id ||= @cloudobj.cloud_id
456
+ end
457
+ res_type = self.class.cfg_plural
458
+ res_name = @config['name'] if !@config.nil?
459
+ @credentials ||= @config['credentials'] if !@config.nil?
460
+ deploydata = nil
461
+
462
+ if !@deploy.nil? and @deploy.is_a?(MU::MommaCat) and
463
+ !@deploy.deployment.nil? and
464
+ !@deploy.deployment[res_type].nil? and
465
+ !@deploy.deployment[res_type][res_name].nil?
466
+ deploydata = @deploy.deployment[res_type][res_name]
467
+ else
468
+ # XXX This should only happen on a brand new resource, but we should
469
+ # probably complain under other circumstances, if we can
470
+ # differentiate them.
471
+ end
472
+
473
+ if self.class.has_multiples and !@mu_name.nil? and deploydata.is_a?(Hash) and deploydata.has_key?(@mu_name)
474
+ @deploydata = deploydata[@mu_name]
475
+ elsif deploydata.is_a?(Hash)
476
+ @deploydata = deploydata
477
+ end
478
+
479
+ if @cloud_id.nil? and @deploydata.is_a?(Hash)
480
+ if @mu_name.nil? and @deploydata.has_key?('#MU_NAME')
481
+ @mu_name = @deploydata['#MU_NAME']
482
+ end
483
+ if @deploydata.has_key?('cloud_id')
484
+ @cloud_id ||= @deploydata['cloud_id']
485
+ end
486
+ end
487
+
488
+ return [@mu_name, @config, @deploydata]
489
+ end
490
+
491
+ # Fetch MU::Cloud objects for each of this object's dependencies, and
492
+ # return in an easily-navigable Hash. This can include things listed in
493
+ # @config['dependencies'], implicitly-defined dependencies such as
494
+ # add_firewall_rules or vpc stanzas, and may refer to objects internal
495
+ # to this deployment or external. Will populate the instance variables
496
+ # @dependencies (general dependencies, which can only be sibling
497
+ # resources in this deployment), as well as for certain config stanzas
498
+ # which can refer to external resources (@vpc, @loadbalancers,
499
+ # @add_firewall_rules)
500
+ def dependencies(use_cache: false, debug: false)
501
+ @dependencies ||= {}
502
+ @loadbalancers ||= []
503
+ @firewall_rules ||= []
504
+
505
+ if @config.nil?
506
+ return [@dependencies, @vpc, @loadbalancers]
507
+ end
508
+ if use_cache and @dependencies.size > 0
509
+ return [@dependencies, @vpc, @loadbalancers]
510
+ end
511
+ @config['dependencies'] = [] if @config['dependencies'].nil?
512
+
513
+ loglevel = debug ? MU::NOTICE : MU::DEBUG
514
+
515
+ # First, general dependencies. These should all be fellow members of
516
+ # the current deployment.
517
+ @config['dependencies'].each { |dep|
518
+ @dependencies[dep['type']] ||= {}
519
+ next if @dependencies[dep['type']].has_key?(dep['name'])
520
+ handle = @deploy.findLitterMate(type: dep['type'], name: dep['name']) if !@deploy.nil?
521
+ if !handle.nil?
522
+ MU.log "Loaded dependency for #{self}: #{dep['name']} => #{handle}", loglevel
523
+ @dependencies[dep['type']][dep['name']] = handle
524
+ else
525
+ # XXX yell under circumstances where we should expect to have
526
+ # our stuff available already?
527
+ end
528
+ }
529
+
530
+ # Special dependencies: my containing VPC
531
+ if self.class.can_live_in_vpc and !@config['vpc'].nil?
532
+ @config['vpc']["id"] ||= @config['vpc']["vpc_id"] # old deploys
533
+ @config['vpc']["name"] ||= @config['vpc']["vpc_name"] # old deploys
534
+ # If something hash-ified a MU::Config::Ref here, fix it
535
+ if !@config['vpc']["id"].nil? and @config['vpc']["id"].is_a?(Hash)
536
+ @config['vpc']["id"] = MU::Config::Ref.new(@config['vpc']["id"])
537
+ end
538
+ if !@config['vpc']["id"].nil?
539
+ if @config['vpc']["id"].is_a?(MU::Config::Ref) and !@config['vpc']["id"].kitten.nil?
540
+ @vpc = @config['vpc']["id"].kitten(@deploy)
541
+ else
542
+ if @config['vpc']['habitat']
543
+ @config['vpc']['habitat'] = MU::Config::Ref.get(@config['vpc']['habitat'])
544
+ end
545
+ vpc_ref = MU::Config::Ref.get(@config['vpc'])
546
+ @vpc = vpc_ref.kitten(@deploy)
547
+ end
548
+ elsif !@config['vpc']["name"].nil? and @deploy
549
+ MU.log "Attempting findLitterMate on VPC for #{self}", loglevel, details: @config['vpc']
550
+
551
+ sib_by_name = @deploy.findLitterMate(name: @config['vpc']['name'], type: "vpcs", return_all: true, habitat: @config['vpc']['project'], debug: debug)
552
+ if sib_by_name.is_a?(Hash)
553
+ if sib_by_name.size == 1
554
+ @vpc = sib_by_name.values.first
555
+ MU.log "Single VPC match for #{self}", loglevel, details: @vpc.to_s
556
+ else
557
+ # XXX ok but this is the wrong place for this really the config parser needs to sort this out somehow
558
+ # we got multiple matches, try to pick one by preferred subnet
559
+ # behavior
560
+ MU.log "Sorting a bunch of VPC matches for #{self}", loglevel, details: sib_by_name.map { |s| s.to_s }.join(", ")
561
+ sib_by_name.values.each { |sibling|
562
+ all_private = sibling.subnets.map { |s| s.private? }.all?(true)
563
+ all_public = sibling.subnets.map { |s| s.private? }.all?(false)
564
+ names = sibling.subnets.map { |s| s.name }
565
+ ids = sibling.subnets.map { |s| s.cloud_id }
566
+ if all_private and ["private", "all_private"].include?(@config['vpc']['subnet_pref'])
567
+ @vpc = sibling
568
+ break
569
+ elsif all_public and ["public", "all_public"].include?(@config['vpc']['subnet_pref'])
570
+ @vpc = sibling
571
+ break
572
+ elsif @config['vpc']['subnet_name'] and
573
+ names.include?(@config['vpc']['subnet_name'])
574
+ #puts "CHOOSING #{@vpc.to_s} 'cause it has #{@config['vpc']['subnet_name']}"
575
+ @vpc = sibling
576
+ break
577
+ elsif @config['vpc']['subnet_id'] and
578
+ ids.include?(@config['vpc']['subnet_id'])
579
+ @vpc = sibling
580
+ break
581
+ end
582
+ }
583
+ if !@vpc
584
+ sibling = sib_by_name.values.sample
585
+ MU.log "Got multiple matching VPCs for #{self.class.cfg_name} #{@mu_name}, so I'm arbitrarily choosing #{sibling.mu_name}", MU::WARN, details: @config['vpc']
586
+ @vpc = sibling
587
+ end
588
+ end
589
+ else
590
+ @vpc = sib_by_name
591
+ MU.log "Found exact VPC match for #{self}", loglevel, details: sib_by_name.to_s
592
+ end
593
+ else
594
+ MU.log "No shortcuts available to fetch VPC for #{self}", loglevel, details: @config['vpc']
595
+ end
596
+
597
+ if !@vpc and !@config['vpc']["name"].nil? and
598
+ @dependencies.has_key?("vpc") and
599
+ @dependencies["vpc"].has_key?(@config['vpc']["name"])
600
+ MU.log "Grabbing VPC I see in @dependencies['vpc']['#{@config['vpc']["name"]}'] for #{self}", loglevel, details: @config['vpc']
601
+ @vpc = @dependencies["vpc"][@config['vpc']["name"]]
602
+ elsif !@vpc
603
+ tag_key, tag_value = @config['vpc']['tag'].split(/=/, 2) if !@config['vpc']['tag'].nil?
604
+ if !@config['vpc'].has_key?("id") and
605
+ !@config['vpc'].has_key?("deploy_id") and !@deploy.nil?
606
+ @config['vpc']["deploy_id"] = @deploy.deploy_id
607
+ end
608
+ MU.log "Doing findStray for VPC for #{self}", loglevel, details: @config['vpc']
609
+ vpcs = MU::MommaCat.findStray(
610
+ @config['cloud'],
611
+ "vpc",
612
+ deploy_id: @config['vpc']["deploy_id"],
613
+ cloud_id: @config['vpc']["id"],
614
+ name: @config['vpc']["name"],
615
+ tag_key: tag_key,
616
+ tag_value: tag_value,
617
+ habitats: [@project_id],
618
+ region: @config['vpc']["region"],
619
+ calling_deploy: @deploy,
620
+ credentials: @credentials,
621
+ dummy_ok: true,
622
+ debug: debug
623
+ )
624
+ @vpc = vpcs.first if !vpcs.nil? and vpcs.size > 0
625
+ end
626
+ if @vpc and @vpc.config and @vpc.config['bastion'] and
627
+ @vpc.config['bastion'].to_h['name'] != @config['name']
628
+ refhash = @vpc.config['bastion'].to_h
629
+ refhash['deploy_id'] ||= @vpc.deploy.deploy_id
630
+ natref = MU::Config::Ref.get(refhash)
631
+ if natref and natref.kitten(@vpc.deploy)
632
+ @nat = natref.kitten(@vpc.deploy)
633
+ end
634
+ end
635
+ if @nat.nil? and !@vpc.nil? and (
636
+ @config['vpc'].has_key?("nat_host_id") or
637
+ @config['vpc'].has_key?("nat_host_tag") or
638
+ @config['vpc'].has_key?("nat_host_ip") or
639
+ @config['vpc'].has_key?("nat_host_name")
640
+ )
641
+
642
+ nat_tag_key, nat_tag_value = @config['vpc']['nat_host_tag'].split(/=/, 2) if !@config['vpc']['nat_host_tag'].nil?
643
+
644
+ @nat = @vpc.findBastion(
645
+ nat_name: @config['vpc']['nat_host_name'],
646
+ nat_cloud_id: @config['vpc']['nat_host_id'],
647
+ nat_tag_key: nat_tag_key,
648
+ nat_tag_value: nat_tag_value,
649
+ nat_ip: @config['vpc']['nat_host_ip']
650
+ )
651
+
652
+ if @nat.nil?
653
+ if !@vpc.cloud_desc.nil?
654
+ @nat = @vpc.findNat(
655
+ nat_cloud_id: @config['vpc']['nat_host_id'],
656
+ nat_filter_key: "vpc-id",
657
+ region: @config['vpc']["region"],
658
+ nat_filter_value: @vpc.cloud_id,
659
+ credentials: @config['credentials']
660
+ )
661
+ else
662
+ @nat = @vpc.findNat(
663
+ nat_cloud_id: @config['vpc']['nat_host_id'],
664
+ region: @config['vpc']["region"],
665
+ credentials: @config['credentials']
666
+ )
667
+ end
668
+ end
669
+ end
670
+ if @vpc.nil? and @config['vpc']
671
+ feck = MU::Config::Ref.get(@config['vpc'])
672
+ feck.kitten(@deploy, debug: true)
673
+ pp feck
674
+ raise MuError.new "#{self.class.cfg_name} #{@config['name']} failed to locate its VPC", details: @config['vpc']
675
+ end
676
+ elsif self.class.cfg_name == "vpc"
677
+ @vpc = self
678
+ end
679
+
680
+ # Google accounts usually have a useful default VPC we can use
681
+ if @vpc.nil? and @project_id and @cloud == "Google" and
682
+ self.class.can_live_in_vpc
683
+ MU.log "Seeing about default VPC for #{self}", MU::NOTICE
684
+ vpcs = MU::MommaCat.findStray(
685
+ "Google",
686
+ "vpc",
687
+ cloud_id: "default",
688
+ habitats: [@project_id],
689
+ credentials: @credentials,
690
+ dummy_ok: true,
691
+ debug: debug
692
+ )
693
+ @vpc = vpcs.first if !vpcs.nil? and vpcs.size > 0
694
+ end
695
+
696
+ # Special dependencies: LoadBalancers I've asked to attach to an
697
+ # instance.
698
+ if @config.has_key?("loadbalancers")
699
+ @loadbalancers = [] if !@loadbalancers
700
+ @config['loadbalancers'].each { |lb|
701
+ MU.log "Loading LoadBalancer for #{self}", MU::DEBUG, details: lb
702
+ if @dependencies.has_key?("loadbalancer") and
703
+ @dependencies["loadbalancer"].has_key?(lb['concurrent_load_balancer'])
704
+ @loadbalancers << @dependencies["loadbalancer"][lb['concurrent_load_balancer']]
705
+ else
706
+ if !lb.has_key?("existing_load_balancer") and
707
+ !lb.has_key?("deploy_id") and !@deploy.nil?
708
+ lb["deploy_id"] = @deploy.deploy_id
709
+ end
710
+ lbs = MU::MommaCat.findStray(
711
+ @config['cloud'],
712
+ "loadbalancer",
713
+ deploy_id: lb["deploy_id"],
714
+ cloud_id: lb['existing_load_balancer'],
715
+ name: lb['concurrent_load_balancer'],
716
+ region: @config["region"],
717
+ calling_deploy: @deploy,
718
+ dummy_ok: true
719
+ )
720
+ @loadbalancers << lbs.first if !lbs.nil? and lbs.size > 0
721
+ end
722
+ }
723
+ end
724
+
725
+ # Munge in external resources referenced by the existing_deploys
726
+ # keyword
727
+ if @config["existing_deploys"] && !@config["existing_deploys"].empty?
728
+ @config["existing_deploys"].each { |ext_deploy|
729
+ if ext_deploy["cloud_id"]
730
+ found = MU::MommaCat.findStray(
731
+ @config['cloud'],
732
+ ext_deploy["cloud_type"],
733
+ cloud_id: ext_deploy["cloud_id"],
734
+ region: @config['region'],
735
+ dummy_ok: false
736
+ ).first
737
+
738
+ MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
739
+ @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
740
+ elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
741
+ MU.log "#{self}: Importing metadata for #{ext_deploy["cloud_type"]} #{ext_deploy["mu_name"]} from #{ext_deploy["deploy_id"]}"
742
+ found = MU::MommaCat.findStray(
743
+ @config['cloud'],
744
+ ext_deploy["cloud_type"],
745
+ deploy_id: ext_deploy["deploy_id"],
746
+ mu_name: ext_deploy["mu_name"],
747
+ region: @config['region'],
748
+ dummy_ok: false
749
+ ).first
750
+
751
+ if found.nil?
752
+ MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR
753
+ else
754
+ @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
755
+ end
756
+ else
757
+ MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
758
+ end
759
+ }
760
+ end
761
+
762
+ if @config['dns_records'] && !@config['dns_records'].empty?
763
+ @config['dns_records'].each { |dnsrec|
764
+ if dnsrec.has_key?("name")
765
+ if dnsrec['name'].start_with?(@deploy.deploy_id.downcase) && !dnsrec['name'].start_with?(@mu_name.downcase)
766
+ MU.log "DNS records for #{@mu_name} seem to be wrong, deleting from current config", MU::WARN, details: dnsrec
767
+ dnsrec.delete('name')
768
+ dnsrec.delete('target')
769
+ end
770
+ end
771
+ }
772
+ end
773
+
774
+ return [@dependencies, @vpc, @loadbalancers]
775
+ end
776
+
777
+ # Using the automatically-defined +@vpc+ from {dependencies} in
778
+ # conjunction with our config, return our configured subnets.
779
+ # @return [Array<MU::Cloud::VPC::Subnet>]
780
+ def mySubnets
781
+ dependencies
782
+ if !@vpc or !@config["vpc"]
783
+ return nil
784
+ end
785
+
786
+ if @config["vpc"]["subnet_id"] or @config["vpc"]["subnet_name"]
787
+ @config["vpc"]["subnets"] ||= []
788
+ subnet_block = {}
789
+ subnet_block["subnet_id"] = @config["vpc"]["subnet_id"] if @config["vpc"]["subnet_id"]
790
+ subnet_block["subnet_name"] = @config["vpc"]["subnet_name"] if @config["vpc"]["subnet_name"]
791
+ @config["vpc"]["subnets"] << subnet_block
792
+ @config["vpc"]["subnets"].uniq!
793
+ end
794
+
795
+ if (!@config["vpc"]["subnets"] or @config["vpc"]["subnets"].empty?) and
796
+ !@config["vpc"]["subnet_id"]
797
+ return @vpc.subnets
798
+ end
799
+
800
+ subnets = []
801
+ @config["vpc"]["subnets"].each { |subnet|
802
+ subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
803
+ raise MuError.new "Couldn't find a live subnet for #{self} matching #{subnet} in #{@vpc}", details: @vpc.subnets.map { |s| s.name }.join(",") if subnet_obj.nil?
804
+ subnets << subnet_obj
805
+ }
806
+
807
+ subnets
808
+ end
809
+
810
+ # @return [Array<MU::Cloud::FirewallRule>]
811
+ def myFirewallRules
812
+ dependencies
813
+
814
+ rules = []
815
+ if @dependencies.has_key?("firewall_rule")
816
+ rules = @dependencies['firewall_rule'].values
817
+ end
818
+ # XXX what other ways are these specified?
819
+
820
+ rules
821
+ end
822
+
823
+ # If applicable, allow this resource's NAT host blanket access via
824
+ # rules in its associated +admin+ firewall rule set.
825
+ def allowBastionAccess
826
+ return nil if !@nat or !@nat.is_a?(MU::Cloud::Server)
827
+
828
+ myFirewallRules.each { |acl|
829
+ if acl.config["admin"]
830
+ acl.addRule(@nat.listIPs, proto: "tcp")
831
+ acl.addRule(@nat.listIPs, proto: "udp")
832
+ acl.addRule(@nat.listIPs, proto: "icmp")
833
+ end
834
+ }
835
+ end
836
+
837
+ # A hook that is always called just before each instance method is
838
+ # invoked, so that we can ensure that repetitive setup tasks (like
839
+ # resolving +:resource_group+ for Azure resources) have always been
840
+ # done.
841
+ def resourceInitHook
842
+ @cloud ||= cloud
843
+ if @cloudparentclass.respond_to?(:resourceInitHook)
844
+ @cloudparentclass.resourceInitHook(@cloudobj, @deploy)
845
+ end
846
+ end
847
+
848
+ if File.exist?(MU.myRoot+"/modules/mu/cloud/#{cfg_name}.rb")
849
+ require "mu/cloud/#{cfg_name}"
850
+ end
851
+
852
+ # Wrap the instance methods that this cloud resource type has to
853
+ # implement.
854
+ MU::Cloud.resource_types[name.to_sym][:instance].each { |method|
855
+
856
+ define_method method do |*args|
857
+ return nil if @cloudobj.nil?
858
+ MU.log "Invoking #{@cloudobj}.#{method}", MU::DEBUG
859
+
860
+ # Go ahead and guarantee that we can't accidentally trigger these
861
+ # methods recursively.
862
+ @method_semaphore.synchronize {
863
+ # We're looking for recursion, not contention, so ignore some
864
+ # obviously harmless things.
865
+ if @method_locks.has_key?(method) and method != :findBastion and method != :cloud_id
866
+ MU.log "Double-call to cloud method #{method} for #{self}", MU::DEBUG, details: caller + ["competing call stack:"] + @method_locks[method]
867
+ end
868
+ @method_locks[method] = caller
869
+ }
870
+
871
+ # Make sure the describe() caches are fresh
872
+ @cloudobj.describe if method != :describe
873
+
874
+ # Don't run through dependencies on simple attr_reader lookups
875
+ if ![:dependencies, :cloud_id, :config, :mu_name].include?(method)
876
+ @cloudobj.dependencies
877
+ end
878
+
879
+ retval = nil
880
+ if !args.nil? and args.size == 1
881
+ retval = @cloudobj.method(method).call(args.first)
882
+ elsif !args.nil? and args.size > 0
883
+ retval = @cloudobj.method(method).call(*args)
884
+ else
885
+ retval = @cloudobj.method(method).call
886
+ end
887
+ if [:create, :groom, :postBoot, :toKitten].include?(method) and
888
+ (!@destroyed and !@cloudobj.destroyed)
889
+ deploydata = @cloudobj.method(:notify).call
890
+ @deploydata ||= deploydata # XXX I don't remember why we're not just doing this from the get-go; maybe because we prefer some mangling occurring in @deploy.notify?
891
+ if deploydata.nil? or !deploydata.is_a?(Hash)
892
+ MU.log "#{self} notify method did not return a Hash of deployment data, attempting to fill in with cloud descriptor #{@cloudobj.cloud_id}", MU::WARN
893
+ deploydata = MU.structToHash(@cloudobj.cloud_desc)
894
+ raise MuError, "Failed to collect metadata about #{self}" if deploydata.nil?
895
+ end
896
+ deploydata['cloud_id'] ||= @cloudobj.cloud_id if !@cloudobj.cloud_id.nil?
897
+ deploydata['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
898
+ deploydata['nodename'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
899
+ deploydata.delete("#MUOBJECT")
900
+ @deploy.notify(self.class.cfg_plural, @config['name'], deploydata, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
901
+ elsif method == :notify
902
+ if retval.nil?
903
+ MU.log self.to_s+" didn't return any metadata from notify", MU::WARN, details: @cloudobj.cloud_desc
904
+ else
905
+ retval['cloud_id'] = @cloudobj.cloud_id.to_s if !@cloudobj.cloud_id.nil?
906
+ retval['mu_name'] = @cloudobj.mu_name if !@cloudobj.mu_name.nil?
907
+ @deploy.notify(self.class.cfg_plural, @config['name'], retval, triggering_node: @cloudobj, delayed_save: @delayed_save) if !@deploy.nil?
908
+ end
909
+ end
910
+ @method_semaphore.synchronize {
911
+ @method_locks.delete(method)
912
+ }
913
+
914
+ @deploydata = @cloudobj.deploydata
915
+ @config = @cloudobj.config
916
+ retval
917
+ end
918
+ } # end instance method list
919
+
920
+
921
+ } # end dynamic class generation block
922
+ } # end resource type iteration
923
+
924
+ require 'mu/cloud/winrm_sessions'
925
+ require 'mu/cloud/ssh_sessions'
926
+
927
+ end
928
+
929
+ end