cloud-mu 3.1.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -105,7 +105,7 @@ if $opts[:dryrun]
105
105
  Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :never) {
106
106
  begin
107
107
  Thread.handle_interrupt(MU::Cloud::MuCloudResourceNotImplemented => :immediate) {
108
- MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
108
+ MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
109
109
  Thread.current.exit
110
110
  }
111
111
  ensure
@@ -124,7 +124,7 @@ if $opts[:dryrun]
124
124
  )
125
125
  cost_dummy_deploy.run
126
126
  rescue MU::Cloud::MuCloudResourceNotImplemented, MU::Cloud::MuCloudFlagNotImplemented
127
- MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::WARN, verbosity: MU::Logger::NORMAL
127
+ MU.log "Cost calculator not available for this stack, as it uses a resource not implemented in Mu's CloudFormation layer.", MU::NOTICE, verbosity: MU::Logger::NORMAL
128
128
  end
129
129
  end
130
130
  exit
@@ -135,7 +135,7 @@ if $opts[:update]
135
135
  # TODO consider whether this is useful/valid
136
136
  # old_conf = JSON.parse(File.read(deploy.deploy_dir+"/basket_of_kittens.json"))
137
137
  # stack_conf = old_conf.merge(stack_conf)
138
- deploy.updateBasketofKittens(stack_conf)
138
+ deploy.updateBasketofKittens(stack_conf, skip_validation: true)
139
139
  deployer = MU::Deploy.new(
140
140
  deploy.environment,
141
141
  verbosity: verbosity,
@@ -0,0 +1,25 @@
1
+ #!/usr/local/ruby-current/bin/ruby
2
+ # Copyright:: Copyright (c) 2014 eGlobalTech, Inc., all rights reserved
3
+ #
4
+ # Licensed under the BSD-3 license (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License in the root of the project or at
7
+ #
8
+ # http://egt-labs.com/mu/LICENSE.html
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'rubygems'
17
+ require 'bundler/setup'
18
+ require 'json'
19
+ require 'erb'
20
+ require 'optimist'
21
+ require 'json-schema'
22
+ require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
23
+ require 'mu'
24
+
25
+ MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, debug: true)
@@ -79,8 +79,7 @@ EOF
79
79
  impl_counts[type] ||= 0
80
80
  [a, b].each { |cloud|
81
81
  begin
82
- myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
83
- case myclass.quality
82
+ case MU::Cloud.resourceClass(cloud, type).quality
84
83
  when MU::Cloud::RELEASE
85
84
  cloud_is_useful[cloud] = true
86
85
  counts[cloud] += 4
@@ -114,8 +113,7 @@ EOF
114
113
  cloudlist.each { |cloud|
115
114
  readme += "<td><center>"
116
115
  begin
117
- myclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(type)
118
- case myclass.quality
116
+ case MU::Cloud.resourceClass(cloud, type).quality
119
117
  when MU::Cloud::RELEASE
120
118
  readme += "<img src='release.png' style='#{icon_style}' title='Release Quality' alt='[Release Quality]'>"
121
119
  when MU::Cloud::BETA
@@ -36,8 +36,7 @@ def validateClouds(cfg)
36
36
  cfg[cloud]["default"]["default"] = true
37
37
  found_default = true
38
38
  else
39
- missing_alias = false
40
- cfg[cloud].each_pair { |acctalias, acct|
39
+ cfg[cloud].values.each { |acct|
41
40
  if acct["default"]
42
41
  if found_default
43
42
  puts "Multiple accounts have 'default' set in #{cloud}"
@@ -135,7 +134,7 @@ def loadMuConfig(default_cfg_overrides = nil)
135
134
  }
136
135
  end
137
136
 
138
- global_cfg = { "config_files" => [] }
137
+ global_cfg = { "config_files" => [], "overridden_keys" => [] }
139
138
  if File.exist?(cfgPath)
140
139
  global_cfg = YAML.load(File.read(cfgPath))
141
140
  global_cfg["config_files"] = [cfgPath]
@@ -148,6 +147,7 @@ def loadMuConfig(default_cfg_overrides = nil)
148
147
  if localfile
149
148
  global_cfg.merge!(localfile)
150
149
  global_cfg["config_files"] << "#{home}/.mu.yaml"
150
+ global_cfg["overridden_keys"] = localfile.keys
151
151
  end
152
152
  end
153
153
  if !global_cfg.has_key?("installdir")
@@ -205,7 +205,7 @@ def cfgPath
205
205
  end
206
206
  end
207
207
  home = Etc.getpwuid(Process.uid).dir
208
- username = Etc.getpwuid(Process.uid).name
208
+ Etc.getpwuid(Process.uid).name # validates existence of a username
209
209
  if Process.uid == 0 and !in_gem
210
210
  if ENV.include?('MU_INSTALLDIR')
211
211
  ENV['MU_INSTALLDIR']+"/etc/mu.yaml"
@@ -29,9 +29,9 @@ Usage:
29
29
  opt :all, "Operate on all nodes/deploys. Use with caution.", :require => false, :default => false, :type => :boolean
30
30
  opt :platform, "Operate exclusively on one nodes of a particular operating system. Can be used in conjunction with -a or -d. Valid platforms: linux, windows", :require => false, :type => :string
31
31
  opt :environment, "Operate exclusively on one nodes with a particular environment (e.g. dev, prod). Can be used in conjunction with -a or -d.", :require => false, :type => :string
32
- opt :override_chef_runlist, "An alternate runlist to pass to Chef, in chefrun mode.", :require => false, :type => :string
32
+ opt :override_chef_runlist, "An alternate runlist to pass to Chef, in groomeronly mode.", :require => false, :type => :string
33
33
  opt :xecute, "Run a shell command on matching nodes. Overrides --mode and suppresses some informational output in favor of scriptability.", :require => false, :type => :string
34
- opt :mode, "Action to perform on matching nodes. Valid actions: groom, chefrun, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "chefrun", :type => :string
34
+ opt :mode, "Action to perform on matching nodes. Valid actions: groom, groomeronly, awsmeta, vaults, certs, chefupgrade", :require => false, :default => "groomeronly", :type => :string
35
35
  opt :verbose, "Show output from Chef runs, etc", :require => false, :default => false, :type => :boolean
36
36
  opt :winrm, "Force WinRM connection. Disable SSH fallback", :require => false, :default => false, :type => :boolean
37
37
  opt :info, "List a particular node attribute", :require => false, :default => 'nodename', :type => :string
@@ -39,8 +39,10 @@ end
39
39
 
40
40
  MU.setLogging(MU::Logger::LOUD) if $opts[:verbose]
41
41
 
42
- if !["groom", "chefrun", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
43
- Optimist::die(:mode, "--mode must be one of: groom, chefrun, awsmeta, vaults, certs, chefupgrade")
42
+ $opts[:mode] = "groomeronly" if $opts[:mode] == "chefrun"
43
+
44
+ if !["groom", "groomeronly", "vaults", "userdata", "awsmeta", "certs", "chefupgrade"].include?($opts[:mode])
45
+ Optimist::die(:mode, "--mode must be one of: groom, groomeronly, awsmeta, vaults, certs, chefupgrade")
44
46
  end
45
47
  if $opts[:platform] and !["linux", "windows"].include?($opts[:platform])
46
48
  Optimist::die(:platform, "--platform must be one of: linux, windows")
@@ -176,7 +178,7 @@ end
176
178
  exit 1 if !ok
177
179
 
178
180
 
179
- def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
181
+ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false, groomeronly: false)
180
182
  badnodes = []
181
183
  count = 0
182
184
  deploys.each { |muid|
@@ -196,6 +198,8 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
196
198
  server.config["vault_access"].each { |v|
197
199
  MU::Groomer::Chef.grantSecretAccess(mu_name, v['vault'], v['item'])
198
200
  }
201
+ elsif groomeronly
202
+ server.groomer.run
199
203
  else
200
204
  mommacat.groomNode(server.cloud_id, nodeclass, type, mu_name: mu_name)
201
205
  end
@@ -227,7 +231,7 @@ def reGroom(deploys = MU::MommaCat.listDeploys, nodes = [], vaults_only: false)
227
231
  end
228
232
  end
229
233
 
230
- def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false, chefrun: false, chef_runlist: nil)
234
+ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_output: $opts[:verbose], noop: false)
231
235
  badnodes = []
232
236
  count = 0
233
237
  deploys.each { |muid|
@@ -247,12 +251,6 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
247
251
  next
248
252
  end
249
253
 
250
- # Generate the command if attemting a chef run
251
- if chefrun
252
- cmd = serverobj.windows? ? "powershell -Command chef-client" : "chef-client || sudo chef-client"
253
- cmd += " -o '#{chef_runlist}'" if chef_runlist
254
- end
255
-
256
254
  MU.log "Running '#{cmd}' on #{nodename} (##{count})" if !print_output
257
255
 
258
256
  # Set Variables to catch the output and exit code of the execution
@@ -363,7 +361,7 @@ def runCommand(deploys = MU::MommaCat.listDeploys, nodes = [], cmd = nil, print_
363
361
  }
364
362
 
365
363
  if badnodes.size > 0
366
- cmd = "Chef" if $opts[:mode] == "chefrun"
364
+ cmd = "Chef" if $opts[:mode] == "groomeronly"
367
365
  if !print_output
368
366
  MU.log "Not all `#{cmd}` runs exited cleanly", MU::WARN, details: badnodes
369
367
  else
@@ -687,12 +685,13 @@ elsif $opts[:mode] == "vaults"
687
685
  reGroom(do_deploys, do_nodes, vaults_only: true)
688
686
  elsif $opts[:mode] == "chefupgrade"
689
687
  chefUpgrade(do_deploys, do_nodes)
690
- elsif $opts[:mode] == "chefrun"
688
+ elsif $opts[:mode] == "groomeronly"
691
689
  print_output = $opts[:verbose] || do_nodes.size == 1
692
690
  if $opts[:override_chef_runlist]
693
- runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], chefrun: true, print_output: print_output)
691
+ # runCommand(do_deploys, do_nodes, chef_runlist: $opts[:override_chef_runlist], groomeronly: true, print_output: print_output)
694
692
  else
695
- runCommand(do_deploys, do_nodes, chefrun: true, print_output: print_output)
693
+ # runCommand(do_deploys, do_nodes, groomeronly: true, print_output: print_output)
694
+ reGroom(do_deploys, do_nodes, groomeronly: true)
696
695
  end
697
696
  elsif $opts[:mode] == "userdata" or $opts[:mode] == "awsmeta"
698
697
  # Need Google equiv and to select nodes correctly based on what cloud they're in
@@ -13,53 +13,151 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
+ bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
17
+ dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
18
+ Dir.chdir(dir)
19
+
16
20
  require 'colorize'
17
21
  require 'optimist'
18
- bindir = File.realpath(File.expand_path(File.dirname(__FILE__)))
19
22
  require bindir+"/mu-load-config.rb"
20
23
  require 'mu'
21
24
 
25
+ require 'rubygems'
26
+ require 'bundler/setup'
27
+ require 'json'
28
+ require 'erb'
29
+ require 'json-schema'
30
+
22
31
  $opts = Optimist::options do
23
32
  banner <<-EOS
24
33
  Usage:
25
- #{$0} [-e <environment>] [-r region] [-v] [-d] [-w] [-c] [-n] [-s] [-j] [-p parameter=value] /path/to/stack/config.[json|yaml] [-u deploy_id [-l]] [-r deploy_id]
34
+ #{$0} [-m <#>] [-f] [-v] [specific test BoK to run [...]]
26
35
  EOS
27
36
  opt :max_threads, "Environment to set on creation.", :require => false, :default => 3, :type => :integer
37
+ opt :max_retries, "Number of times to retry failed tests in --dryrun mode.", :require => false, :default => 2, :type => :integer
28
38
  opt :full, "Actually run deploys, instead of --dryrun", :require => false, :default => false
39
+ opt :verbose, "Show more information while running", :require => false, :default => false
29
40
  end
30
41
 
31
- dir = File.realpath(File.expand_path(bindir+"/../modules/tests"))
32
- Dir.chdir(dir)
42
+ only = ARGV
33
43
 
34
44
  files = Dir.glob("*.yaml", base: dir)
35
45
  files.concat(Dir.glob("*.yml", base: dir))
36
- baseclouds = MU::Cloud.supportedClouds.reject { |c| c == "CloudFormation" }
46
+ baseclouds = MU::Cloud.availableClouds.reject { |c| c == "CloudFormation" }
37
47
 
38
48
  commands = {}
49
+ failures = []
50
+ if only and only.size > 0
51
+ files.reject! { |f| !only.include?(f) }
52
+ if files.size == 0
53
+ MU.log "No files in #{dir} matched requested list", MU::WARN, details: only
54
+ exit 1
55
+ end
56
+ end
39
57
 
40
58
  files.each { |f|
41
59
  clouds = baseclouds.dup
60
+ groomer_match = true
42
61
  File.open(dir+"/"+f).readlines.each { |l|
43
62
  l.chomp!
44
- next if !l.match(/^\s*#\s*clouds: (.*)/)
45
- clouds = []
46
- cloudstr = Regexp.last_match[1]
47
- cloudstr.split(/\s*,\s*/).each { |c|
48
- baseclouds.each { |cloud|
49
- if cloud.match(/^#{Regexp.quote(c)}$/i)
50
- clouds << cloud
63
+ if l.match(/^\s*#\s*clouds: (.*)/)
64
+ clouds = []
65
+ cloudstr = Regexp.last_match[1]
66
+ cloudstr.split(/\s*,\s*/).each { |c|
67
+ baseclouds.each { |cloud|
68
+ if cloud.match(/^#{Regexp.quote(c)}$/i)
69
+ clouds << cloud
70
+ end
71
+ }
72
+ }
73
+ elsif l.match(/^\s*#\s*groomers: (.*)/)
74
+ groomerstr = Regexp.last_match[1]
75
+ groomerstr.split(/\s*,\s*/).each { |g|
76
+ if !MU::Groomer.availableGroomers.include?(g)
77
+ MU.log "#{f} requires groomer #{g}, which is not available. This test will be skipped.", MU::NOTICE
78
+ groomer_match = false
51
79
  end
52
80
  }
53
- }
54
- break
81
+ end
55
82
  }
83
+ if !groomer_match
84
+ next
85
+ end
86
+
56
87
  clouds.each { |cloud|
57
- commands["mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"] = { "file" => f, "cloud" => cloud }
88
+ cmd = "mu-deploy #{f} --cloud #{cloud} #{$opts[:full] ? "" : "--dryrun"}"
89
+ commands[cmd] = {
90
+ "file" => f,
91
+ "cloud" => cloud,
92
+ }
93
+ if $opts[:full]
94
+ $environment = "dev"
95
+ begin
96
+ conf_engine = MU::Config.new(f, cloud: cloud)
97
+ rescue StandardError => e
98
+ MU.log e.message+" parsing "+f+" with cloud "+cloud, MU::WARN, details: e.backtrace
99
+ failures << f+" ("+commands[cmd]["cloud"]+")"
100
+ next
101
+ end
102
+ parsed = MU::Config.stripConfig(conf_engine.config)
103
+ types = []
104
+ MU::Cloud.resource_types.values.each { |attrs|
105
+ if parsed.has_key?(attrs[:cfg_plural])
106
+ types << attrs[:cfg_plural]
107
+ end
108
+ }
109
+ commands[cmd]["parsed"] = parsed
110
+ commands[cmd]["types"] = types
111
+ end
58
112
  }
59
113
  }
60
114
 
61
115
  puts "Running #{commands.size.to_s.bold} #{$opts[:full] ? "full deploy" : "parse"} tests from #{files.size.to_s.bold} Baskets of Kittens across #{baseclouds.size.to_s.bold} clouds"
62
116
 
117
+ @output_semaphore = Mutex.new
118
+
119
+ def execCommand(cmd, results_stash)
120
+ @output_semaphore.synchronize {
121
+ puts cmd if $opts[:verbose]
122
+ }
123
+
124
+ ok = true
125
+ retries = 0
126
+ begin
127
+ output = %x{#{cmd} 2>&1}
128
+ if $?.exitstatus != 0
129
+ ok = false
130
+ retries += 1
131
+ if $opts[:verbose] and !$opts[:full] and retries <= $opts[:max_retries]
132
+ puts "#{cmd} RETRY #{retries.to_s}".light_red
133
+ end
134
+ else
135
+ ok = true
136
+ end
137
+ end while !ok and !$opts[:full] and retries <= $opts[:max_retries]
138
+
139
+ results_stash["output"] += output
140
+
141
+ @output_semaphore.synchronize {
142
+ if ok
143
+ if $opts[:verbose]
144
+ puts "#{cmd} SUCCEEDED".green
145
+ else
146
+ print ".".green
147
+ end
148
+ else
149
+ if $opts[:verbose]
150
+ puts "#{cmd} FAILED:".light_red
151
+ puts output
152
+ else
153
+ print ".".light_red
154
+ end
155
+ end
156
+ }
157
+
158
+ ok
159
+ end
160
+
63
161
  threads = []
64
162
  results = {}
65
163
  commands.keys.each { |cmd|
@@ -72,21 +170,40 @@ commands.keys.each { |cmd|
72
170
  end
73
171
 
74
172
  threads << Thread.new(cmd) { |cmd_thr|
75
- results[cmd_thr] = {}
76
- puts %Q{#{cmd_thr} 2>&1}
77
- results[cmd_thr]["output"] = %x{#{cmd_thr} 2>&1}
78
- if $?.exitstatus != 0
79
- results[cmd_thr]["status"] = false
80
- print ".".light_red
81
- else
82
- results[cmd_thr]["status"] = true
83
- print ".".green
173
+ results[cmd_thr] = { "output" => "", "failed" => [] }
174
+ if !execCommand(cmd_thr, results[cmd_thr])
175
+ results[cmd_thr]["failed"] << "main"
84
176
  end
177
+
85
178
  if $opts[:full] and results[cmd_thr]["output"].match(/deploy - Deployment id: .*? \((.*?)\)/)
86
179
  deploy_id = Regexp.last_match[1]
87
- results[cmd_thr]["output"] += %x{mu-cleanup #{deploy_id} --skipsnapshots}
88
- if $?.exitstatus != 0
89
- results[cmd_thr]["status"] = false
180
+ adoptdir = Dir.mktmpdir(commands[cmd_thr]["file"].gsub(/[^a-z0-9]|yaml$/i, ""))
181
+ if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
182
+ adopt = "cd #{adoptdir} && mu-adopt --appname adoptone --grouping omnibus --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
183
+ if !execCommand(adopt, results[cmd_thr])
184
+ results[cmd_thr]["failed"] << "adopt"
185
+ end
186
+ end
187
+
188
+ if File.exist?(dir+"/regrooms/"+commands[cmd_thr]["file"])
189
+ regroom = "mu-deploy regrooms/#{commands[cmd_thr]["file"]} --cloud #{commands[cmd_thr]["cloud"]} --update #{deploy_id} 2>&1"
190
+ if !execCommand(regroom, results[cmd_thr])
191
+ results[cmd_thr]["failed"] << "regroom"
192
+ end
193
+ if commands[cmd_thr]["types"] and commands[cmd_thr]["types"].size > 0
194
+ re_adopt = "cd #{adoptdir} && mu-adopt --appname adopttwo --grouping omnibus --clouds #{commands[cmd_thr]["cloud"]} --types #{commands[cmd_thr]["types"].join(" ")} 2>&1"
195
+ if !execCommand(re_adopt, results[cmd_thr])
196
+ results[cmd_thr]["failed"] << "second adopt"
197
+ end
198
+ end
199
+ # TODO big flex is to read back both adopted BoKs and .diff them, but without
200
+ # all resources having implemented adoption this isn't much of a test yet
201
+ end
202
+
203
+ FileUtils.remove_entry(adoptdir)
204
+ cleanup = %Q{mu-cleanup #{deploy_id} --skipsnapshots}
205
+ if !execCommand(cleanup, results[cmd_thr])
206
+ results[cmd_thr]["failed"] << "cleanup"
90
207
  end
91
208
  end
92
209
  }
@@ -96,21 +213,14 @@ threads.each { |t|
96
213
  }
97
214
  puts ""
98
215
 
99
- failures = []
100
216
  results.keys.sort { |a, b|
101
- if results[b]["status"] and !results[a]["status"]
102
- 1
103
- elsif !results[b]["status"] and results[a]["status"]
104
- -1
105
- else
106
- 0
107
- end
217
+ results[b]["failed"].size <=> results[a]["failed"].size
108
218
  }.each { |cmd|
109
- if !results[cmd]["status"]
219
+ if results[cmd]["failed"].size > 0
110
220
  puts cmd+" failed:".light_red
111
221
  puts results[cmd]["output"].yellow
112
222
  puts "^ #{cmd}".light_red
113
- failures << commands[cmd]["file"]+" ("+commands[cmd]["cloud"]+")"
223
+ failures << commands[cmd]["file"]+" in "+commands[cmd]["cloud"]+" ("+results[cmd]["failed"].join(", ")+")"
114
224
  else
115
225
  puts cmd+" passed".green
116
226
  end
@@ -17,8 +17,8 @@ end
17
17
 
18
18
  Gem::Specification.new do |s|
19
19
  s.name = 'cloud-mu'
20
- s.version = '3.1.3'
21
- s.date = '2020-01-31'
20
+ s.version = '3.3.0'
21
+ s.date = '2020-09-22'
22
22
  s.require_paths = ['modules']
23
23
  s.required_ruby_version = '>= 2.4'
24
24
  s.summary = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -35,31 +35,33 @@ EOF
35
35
  s.homepage =
36
36
  'https://github.com/cloudamatic/mu'
37
37
  s.license = 'BSD-3-Clause-Attribution'
38
- s.add_runtime_dependency 'erubis', "~> 2.7"
39
- s.add_runtime_dependency 'inifile', "~> 3.0"
40
- s.add_runtime_dependency 'bundler', "~> 1.17"
41
- s.add_runtime_dependency 'yard', "~> 0.9"
42
- s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
38
+ s.add_runtime_dependency 'addressable', '~> 2.5'
43
39
  s.add_runtime_dependency "aws-sdk-core", "< 3"
40
+ s.add_runtime_dependency 'azure_sdk', "~> 0.52"
41
+ s.add_runtime_dependency 'bundler', "~> 1.17"
44
42
  s.add_runtime_dependency 'chronic_duration', "~> 0.10"
45
- s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
46
- s.add_runtime_dependency 'optimist', "~> 3.0"
47
- s.add_runtime_dependency 'json-schema', "~> 2.8"
48
- s.add_runtime_dependency 'colorize', "~> 0.8"
49
43
  s.add_runtime_dependency 'color', "~> 1.8"
50
- s.add_runtime_dependency 'netaddr', '~> 2.0'
51
- s.add_runtime_dependency 'nokogiri', "~> 1.10"
52
- s.add_runtime_dependency 'solve', '~> 4.0'
44
+ s.add_runtime_dependency 'colorize', "~> 0.8"
45
+ s.add_runtime_dependency 'erubis', "~> 2.7"
46
+ s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
47
+ s.add_runtime_dependency 'googleauth', "~> 0.6"
48
+ s.add_runtime_dependency 'inifile', "~> 3.0"
49
+ s.add_runtime_dependency 'json-schema', "~> 2.8"
53
50
  s.add_runtime_dependency 'net-ldap', "~> 0.16"
54
51
  s.add_runtime_dependency 'net-ssh', "~> 4.2"
55
52
  s.add_runtime_dependency 'net-ssh-multi', '~> 1.2', '>= 1.2.1'
56
- s.add_runtime_dependency 'googleauth', "~> 0.6"
57
- s.add_runtime_dependency 'google-api-client', "~> 0.36.4"
53
+ s.add_runtime_dependency 'netaddr', '~> 2.0'
54
+ s.add_runtime_dependency 'nokogiri', "~> 1.10"
55
+ s.add_runtime_dependency 'openssl-oaep', "~> 0.1"
56
+ s.add_runtime_dependency 'optimist', "~> 3.0"
57
+ s.add_runtime_dependency 'rack', "~> 2.0"
58
+ s.add_runtime_dependency 'ruby-graphviz', "~> 1.2"
58
59
  s.add_runtime_dependency 'rubocop', '~> 0.58'
59
- s.add_runtime_dependency 'addressable', '~> 2.5'
60
+ s.add_runtime_dependency 'rubyzip', "~> 2.3"
61
+ s.add_runtime_dependency 'simple-password-gen', "~> 0.1"
60
62
  s.add_runtime_dependency 'slack-notifier', "~> 2.3"
61
- s.add_runtime_dependency 'azure_sdk', "~> 0.37"
62
- s.add_runtime_dependency 'rack', "~> 2.0"
63
+ s.add_runtime_dependency 'solve', '~> 4.0'
63
64
  s.add_runtime_dependency 'thin', "~> 1.7"
64
- s.add_runtime_dependency 'rubyzip', "~> 2.0"
65
+ s.add_runtime_dependency 'winrm', "~> 2.3", ">= 2.3.4"
66
+ s.add_runtime_dependency 'yard', "~> 0.9"
65
67
  end