cloud-mu 3.1.4 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +16 -12
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -1
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +37 -12
  23. data/cloud-mu.gemspec +5 -3
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  27. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  28. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  29. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  30. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  31. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  32. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  33. data/extras/clean-stock-amis +25 -19
  34. data/extras/generate-stock-images +1 -0
  35. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  36. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  37. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  38. data/modules/mommacat.ru +1 -1
  39. data/modules/mu.rb +158 -107
  40. data/modules/mu/adoption.rb +386 -59
  41. data/modules/mu/cleanup.rb +214 -303
  42. data/modules/mu/cloud.rb +128 -1632
  43. data/modules/mu/cloud/database.rb +49 -0
  44. data/modules/mu/cloud/dnszone.rb +44 -0
  45. data/modules/mu/cloud/machine_images.rb +212 -0
  46. data/modules/mu/cloud/providers.rb +81 -0
  47. data/modules/mu/cloud/resource_base.rb +926 -0
  48. data/modules/mu/cloud/server.rb +40 -0
  49. data/modules/mu/cloud/server_pool.rb +1 -0
  50. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  51. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  52. data/modules/mu/cloud/wrappers.rb +169 -0
  53. data/modules/mu/config.rb +135 -82
  54. data/modules/mu/config/alarm.rb +2 -6
  55. data/modules/mu/config/bucket.rb +32 -3
  56. data/modules/mu/config/cache_cluster.rb +2 -2
  57. data/modules/mu/config/cdn.rb +100 -0
  58. data/modules/mu/config/collection.rb +1 -1
  59. data/modules/mu/config/container_cluster.rb +7 -2
  60. data/modules/mu/config/database.rb +84 -105
  61. data/modules/mu/config/database.yml +1 -2
  62. data/modules/mu/config/dnszone.rb +5 -4
  63. data/modules/mu/config/doc_helpers.rb +5 -6
  64. data/modules/mu/config/endpoint.rb +2 -1
  65. data/modules/mu/config/firewall_rule.rb +3 -19
  66. data/modules/mu/config/folder.rb +1 -1
  67. data/modules/mu/config/function.rb +17 -8
  68. data/modules/mu/config/group.rb +1 -1
  69. data/modules/mu/config/habitat.rb +1 -1
  70. data/modules/mu/config/job.rb +89 -0
  71. data/modules/mu/config/loadbalancer.rb +57 -11
  72. data/modules/mu/config/log.rb +1 -1
  73. data/modules/mu/config/msg_queue.rb +1 -1
  74. data/modules/mu/config/nosqldb.rb +1 -1
  75. data/modules/mu/config/notifier.rb +8 -19
  76. data/modules/mu/config/ref.rb +92 -14
  77. data/modules/mu/config/role.rb +1 -1
  78. data/modules/mu/config/schema_helpers.rb +38 -37
  79. data/modules/mu/config/search_domain.rb +1 -1
  80. data/modules/mu/config/server.rb +12 -13
  81. data/modules/mu/config/server.yml +1 -0
  82. data/modules/mu/config/server_pool.rb +3 -7
  83. data/modules/mu/config/storage_pool.rb +1 -1
  84. data/modules/mu/config/tail.rb +11 -0
  85. data/modules/mu/config/user.rb +1 -1
  86. data/modules/mu/config/vpc.rb +27 -23
  87. data/modules/mu/config/vpc.yml +0 -1
  88. data/modules/mu/defaults/AWS.yaml +91 -68
  89. data/modules/mu/defaults/Azure.yaml +1 -0
  90. data/modules/mu/defaults/Google.yaml +1 -0
  91. data/modules/mu/deploy.rb +33 -19
  92. data/modules/mu/groomer.rb +16 -1
  93. data/modules/mu/groomers/ansible.rb +123 -21
  94. data/modules/mu/groomers/chef.rb +64 -11
  95. data/modules/mu/logger.rb +120 -144
  96. data/modules/mu/master.rb +97 -4
  97. data/modules/mu/master/ssl.rb +0 -1
  98. data/modules/mu/mommacat.rb +154 -867
  99. data/modules/mu/mommacat/daemon.rb +23 -14
  100. data/modules/mu/mommacat/naming.rb +110 -3
  101. data/modules/mu/mommacat/search.rb +495 -0
  102. data/modules/mu/mommacat/storage.rb +225 -192
  103. data/modules/mu/{clouds → providers}/README.md +1 -1
  104. data/modules/mu/{clouds → providers}/aws.rb +281 -64
  105. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  106. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  107. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  108. data/modules/mu/providers/aws/cdn.rb +782 -0
  109. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  110. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
  111. data/modules/mu/providers/aws/database.rb +1744 -0
  112. data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
  113. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  114. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
  115. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  116. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  117. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  118. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  119. data/modules/mu/providers/aws/job.rb +466 -0
  120. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
  121. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  122. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  123. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  124. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  125. data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
  126. data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
  127. data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
  128. data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
  129. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  130. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  131. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  132. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  133. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  134. data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
  135. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  136. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  137. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  138. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  139. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  140. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  141. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  142. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  143. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  144. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  145. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  146. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  147. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  148. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  149. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  150. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  151. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  152. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  153. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  156. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  158. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  159. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  160. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  161. data/modules/mu/{clouds → providers}/google.rb +29 -6
  162. data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
  163. data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
  164. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  165. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  166. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  167. data/modules/mu/{clouds → providers}/google/function.rb +14 -8
  168. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  169. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  170. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  171. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  172. data/modules/mu/{clouds → providers}/google/server.rb +142 -55
  173. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  174. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  175. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  176. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  177. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  178. data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
  179. data/modules/tests/aws-jobs-functions.yaml +46 -0
  180. data/modules/tests/centos6.yaml +15 -0
  181. data/modules/tests/centos7.yaml +15 -0
  182. data/modules/tests/centos8.yaml +12 -0
  183. data/modules/tests/ecs.yaml +23 -0
  184. data/modules/tests/eks.yaml +1 -1
  185. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  186. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/microservice_app.yaml +288 -0
  189. data/modules/tests/rds.yaml +108 -0
  190. data/modules/tests/regrooms/rds.yaml +123 -0
  191. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  192. data/modules/tests/super_complex_bok.yml +2 -2
  193. data/modules/tests/super_simple_bok.yml +3 -5
  194. data/modules/tests/win2k12.yaml +25 -0
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +169 -93
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1974
  202. data/modules/mu/clouds/aws/endpoint.rb +0 -596
  203. data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -195,48 +195,49 @@ action :config do
195
195
  else
196
196
  # We want to run ec2config as admin user so Windows userdata executes as admin, however the local admin account doesn't have Logon As a Service right. Domain privileges are set separately
197
197
 
198
- cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
199
- source "ntrights"
200
- end
201
- [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
202
- pass = if usr == new_resource.ec2config_user
203
- new_resource.ec2config_password
204
- elsif usr == new_resource.ssh_user
205
- new_resource.ssh_password
206
- end
207
-
208
- user usr do
209
- password pass
210
- end
211
-
212
- group "Administrators" do
213
- action :modify
214
- members usr
215
- append true
216
- end
217
-
218
- %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
219
- batch "Grant local user #{usr} logon as service right" do
220
- code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
221
- end
222
- }
223
-
224
- # XXX user resource seems not to really be setting password, or is setting # in such a way that the user is being required to change it. Workaround.
225
- powershell_script "Adjust local account params for #{usr}" do
226
- code <<-EOH
227
- (([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
228
- EOH
229
- end
230
-
231
- if usr == new_resource.ssh_user
232
-
233
- %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
234
- batch "Grant local user #{usr} logon as service right" do
235
- code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
236
- end
237
- }
238
-
239
- end
240
- }
198
+ # cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
199
+ # source "ntrights"
200
+ # end
201
+ # [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
202
+ # pass = if usr == new_resource.ec2config_user
203
+ # new_resource.ec2config_password
204
+ # elsif usr == new_resource.ssh_user
205
+ # new_resource.ssh_password
206
+ # end
207
+ #
208
+ # user usr do
209
+ # password pass
210
+ # action :modify
211
+ # end
212
+ #
213
+ # group "Administrators" do
214
+ # action :modify
215
+ # members usr
216
+ # append true
217
+ # end
218
+ #
219
+ # %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
220
+ # batch "Grant local user #{usr} logon as service right" do
221
+ # code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
222
+ # end
223
+ # }
224
+ #
225
+ # # XXX user resource seems not to really be setting password, or is setting # in such a way that the user is being required to change it. Workaround.
226
+ # powershell_script "Adjust local account params for #{usr}" do
227
+ # code <<-EOH
228
+ # (([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
229
+ # EOH
230
+ # end
231
+ #
232
+ # if usr == new_resource.ssh_user
233
+ #
234
+ # %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
235
+ # batch "Grant local user #{usr} logon as service right" do
236
+ # code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
237
+ # end
238
+ # }
239
+ #
240
+ # end
241
+ # }
241
242
  end
242
243
  end
@@ -18,37 +18,43 @@ require 'json'
18
18
  require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
19
19
  require 'mu'
20
20
 
21
- credentials = if ARGV[0] and !ARGV[0].empty?
22
- ARGV[0]
23
- else
24
- nil
21
+ $opts = Optimist::options do
22
+ banner <<-EOS
23
+ #{$0} [-c credentials] [-i imagename]
24
+ EOS
25
+ opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
26
+ opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
25
27
  end
26
28
 
27
29
  filters = [
28
30
  {
29
31
  name: "owner-id",
30
- values: [MU::Cloud::AWS.credToAcct(credentials)]
32
+ values: [MU::Cloud::AWS.credToAcct($opts[:credentials])]
31
33
  }
32
34
  ]
33
35
 
34
36
 
35
37
  MU::Cloud::AWS.listRegions.each { | r|
36
- images = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_images(
38
+ images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
37
39
  filters: filters + [{ "name" => "state", "values" => ["available"]}]
38
40
  ).images
39
41
  images.each { |ami|
40
- if (DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/)
41
- snaps = []
42
- ami.block_device_mappings.each { |dev|
43
- if !dev.ebs.nil?
44
- snaps << dev.ebs.snapshot_id
45
- end
46
- }
47
- MU.log "Deregistering #{ami.name} (#{ami.creation_date})", MU::WARN, details: snaps
48
- MU::Cloud::AWS.ec2(region: r, credentials: credentials).deregister_image(image_id: ami.image_id)
49
- snaps.each { |snap_id|
50
- MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
51
- }
52
- end
42
+ if ($opts[:image] and ami.name == $opts[:image]) or
43
+ ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
44
+ snaps = []
45
+ ami.block_device_mappings.each { |dev|
46
+ if !dev.ebs.nil?
47
+ snaps << dev.ebs.snapshot_id
48
+ end
49
+ }
50
+ MU.log "Deregistering #{ami.name}, #{r} (#{ami.creation_date})", MU::WARN, details: snaps
51
+ begin
52
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).deregister_image(image_id: ami.image_id)
53
+ rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
54
+ end
55
+ snaps.each { |snap_id|
56
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
57
+ }
58
+ end
53
59
  }
54
60
  }
@@ -91,6 +91,7 @@ $opts[:clouds].each { |cloud|
91
91
  end
92
92
  next if !needed
93
93
  end
94
+ MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
94
95
  conf_engine = MU::Config.new(
95
96
  bok_dir+"/"+cloud+"/"+platform+".yaml",
96
97
  default_credentials: $opts[(cloud.downcase+"_creds").to_sym]
@@ -1,16 +1,21 @@
1
1
  ---
2
2
  appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
3
5
  servers:
4
- -
5
- name: win2k12
6
- platform: windows
7
- size: m4.large
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::updates]
11
- - recipe[mu-utility::cleanup_image_helper]
12
- create_image:
13
- image_then_destroy: true
14
- public: true
15
- copy_to_regions:
16
- - "#ALL"
6
+ - name: win2k12
7
+ platform: win2k12
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -1,16 +1,21 @@
1
1
  ---
2
2
  appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
3
5
  servers:
4
- -
5
- name: win2k16
6
- platform: windows
7
- size: m4.large
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::updates]
11
- - recipe[mu-utility::cleanup_image_helper]
12
- create_image:
13
- image_then_destroy: true
14
- public: true
15
- copy_to_regions:
16
- - "#ALL"
6
+ - name: win2k16
7
+ platform: win2k16
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -0,0 +1,21 @@
1
+ ---
2
+ appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
5
+ servers:
6
+ - name: win2k19
7
+ platform: windows
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -51,7 +51,7 @@ Signal.trap("URG") do
51
51
  end
52
52
 
53
53
  begin
54
- MU::MommaCat.syncMonitoringConfig(false)
54
+ MU::Master.syncMonitoringConfig(false)
55
55
  rescue StandardError => e
56
56
  MU.log e.inspect, MU::ERR, details: e.backtrace
57
57
  # ...but don't die!
@@ -79,38 +79,40 @@ class Hash
79
79
  }
80
80
  return 0 if self == other # that was easy!
81
81
  # compare elements and decide who's "bigger" based on their totals?
82
- 0
82
+
83
+ # fine, try some brute force and just hope everything implements to_s
84
+ self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
83
85
  end
84
86
 
85
- # Recursively compare two hashes
86
- def diff(with, on = self, level: 0, parents: [])
87
+ # Recursively compare two Mu Basket of Kittens hashes and report the differences
88
+ def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
87
89
  return if with.nil? and on.nil?
88
90
  if with.nil? or on.nil? or with.class != on.class
89
91
  return # XXX ...however we're flagging differences
90
92
  end
91
93
  return if on == with
92
94
 
93
- tree = ""
94
- indentsize = 0
95
- parents.each { |p|
96
- tree += (" " * indentsize) + p + " => \n"
97
- indentsize += 2
98
- }
99
- indent = (" " * indentsize)
100
-
101
95
  changes = []
96
+ report ||= {}
102
97
  if on.is_a?(Hash)
103
98
  on_unique = (on.keys - with.keys)
104
99
  with_unique = (with.keys - on.keys)
105
100
  shared = (with.keys & on.keys)
106
101
  shared.each { |k|
107
- diff(with[k], on[k], level: level+1, parents: parents + [k])
102
+
103
+ report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
104
+ if report_data and !report_data.empty?
105
+ report ||= {}
106
+ report[k] = report_data
107
+ end
108
108
  }
109
109
  on_unique.each { |k|
110
- changes << "- ".red+PP.pp({k => on[k] }, '')
110
+ report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
111
+ report[k][:habitat] = habitat if habitat
111
112
  }
112
113
  with_unique.each { |k|
113
- changes << "+ ".green+PP.pp({k => with[k]}, '')
114
+ report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
115
+ report[k][:habitat] = habitat if habitat
114
116
  }
115
117
  elsif on.is_a?(Array)
116
118
  return if with == on
@@ -122,29 +124,27 @@ class Hash
122
124
  # sorting arrays full of weird, non-primitive types.
123
125
  done = []
124
126
  on.sort.each { |elt|
125
- if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
126
- with.sort.each { |other_elt|
127
- # Figure out what convention this thing is using for resource identification
128
- compare_a, compare_b = if elt['name'].nil? and elt["id"].nil? and !elt["entity"].nil? and !other_elt["entity"].nil?
129
- [elt["entity"], other_elt["entity"]]
130
- else
131
- [elt, other_elt]
132
- end
127
+ if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
128
+ elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
133
129
 
134
- if (compare_a['name'] and compare_b['name'] == compare_a['name']) or
135
- (compare_a['name'].nil? and !compare_a["id"].nil? and compare_a["id"] == compare_b["id"])
136
- break if elt == other_elt
130
+ with.sort.each { |other_elt|
131
+ other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
132
+
133
+ # Case 1: The array element exists in both version of this array
134
+ if elt_namestr and other_elt_namestr and
135
+ elt_namestr == other_elt_namestr and
136
+ (elt_location.nil? or other_elt_location.nil? or
137
+ elt_location == other_elt_location or
138
+ !(elt_location_list & other_elt_location_list).empty?
139
+ )
137
140
  done << elt
138
141
  done << other_elt
139
- namestr = if elt['type']
140
- "#{elt['type']}[#{elt['name']}]"
141
- elsif elt['name']
142
- elt['name']
143
- elsif elt['entity'] and elt["entity"]["id"]
144
- elt['entity']['id']
142
+ break if elt == other_elt # if they're identical, we're done
143
+ report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
144
+ if report_data and !report_data.empty?
145
+ report ||= {}
146
+ report[elt_namestr] = report_data
145
147
  end
146
-
147
- diff(other_elt, elt, level: level+1, parents: parents + [namestr])
148
148
  break
149
149
  end
150
150
  }
@@ -152,43 +152,34 @@ class Hash
152
152
  }
153
153
  on_unique = (on - with) - done
154
154
  with_unique = (with - on) - done
155
- # if on_unique.size > 0 or with_unique.size > 0
156
- # if before_a != after_a
157
- # MU.log "A BEFORE", MU::NOTICE, details: before_a
158
- # MU.log "A AFTER", MU::NOTICE, details: after_a
159
- # end
160
- # if before_b != after_b
161
- # MU.log "B BEFORE", MU::NOTICE, details: before_b
162
- # MU.log "B AFTER", MU::NOTICE, details: after_b
163
- # end
164
- # end
155
+
156
+ # Case 2: This array entry exists in the old version, but not the new one
165
157
  on_unique.each { |e|
166
- changes << if e.is_a?(Hash)
167
- "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n "+(indent))
168
- else
169
- "- ".red+e.to_s
170
- end
158
+ namestr, loc = MU::MommaCat.getChunkName(e)
159
+
160
+ report ||= {}
161
+ report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
162
+ report[namestr][:habitat] = loc if loc
171
163
  }
164
+
165
+ # Case 3: This array entry exists in the new version, but not the old one
172
166
  with_unique.each { |e|
173
- changes << if e.is_a?(Hash)
174
- "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n "+(indent))
175
- else
176
- "+ ".green+e.to_s
177
- end
167
+ namestr, loc = MU::MommaCat.getChunkName(e)
168
+
169
+ report ||= {}
170
+ report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
171
+ report[namestr][:habitat] = loc if loc
178
172
  }
173
+
174
+ # A plain old leaf node of data
179
175
  else
180
176
  if on != with
181
- changes << "-".red+" #{on.to_s}"
182
- changes << "+".green+" #{with.to_s}"
177
+ report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
178
+ report[:habitat] = habitat if habitat
183
179
  end
184
180
  end
185
181
 
186
- if changes.size > 0
187
- puts tree
188
- changes.each { |c|
189
- puts indent+c
190
- }
191
- end
182
+ report.freeze
192
183
  end
193
184
 
194
185
  # Implement a merge! that just updates each hash leaf as needed, not
@@ -212,8 +203,29 @@ class Hash
212
203
  end
213
204
 
214
205
  ENV['HOME'] = Etc.getpwuid(Process.uid).dir
206
+ module MU
207
+
208
+ # For log entries that should only be logged when we're in verbose mode
209
+ DEBUG = 0.freeze
210
+ # For ordinary log entries
211
+ INFO = 1.freeze
212
+ # For more interesting log entries which are not errors
213
+ NOTICE = 2.freeze
214
+ # Log entries for non-fatal errors
215
+ WARN = 3.freeze
216
+ # Log entries for non-fatal errors
217
+ WARNING = 3.freeze
218
+ # Log entries for fatal errors
219
+ ERR = 4.freeze
220
+ # Log entries for fatal errors
221
+ ERROR = 4.freeze
222
+ # Log entries that will be held and displayed/emailed at the end of deploy,
223
+ # cleanup, etc.
224
+ SUMMARY = 5.freeze
225
+ end
215
226
 
216
227
  require 'mu/logger'
228
+
217
229
  module MU
218
230
 
219
231
  # Subclass core thread so we can gracefully handle it when we hit system
@@ -263,6 +275,7 @@ module MU
263
275
  end while newguy.nil?
264
276
 
265
277
  @@mu_global_thread_semaphore.synchronize {
278
+ MU.dupGlobals(Thread.current.object_id, target_thread: newguy)
266
279
  @@mu_global_threads << newguy
267
280
  }
268
281
 
@@ -272,8 +285,9 @@ module MU
272
285
  # Wrapper class for fatal Exceptions. Gives our internals something to
273
286
  # inherit that will log an error message appropriately before bubbling up.
274
287
  class MuError < StandardError
275
- def initialize(message = nil)
276
- MU.log message, MU::ERR, details: caller[2] if !message.nil?
288
+ def initialize(message = nil, silent: false, details: nil)
289
+ details ||= caller[2]
290
+ MU.log message, MU::ERR, details: details if !message.nil? and !silent
277
291
  if MU.verbosity == MU::Logger::SILENT
278
292
  super ""
279
293
  else
@@ -285,8 +299,8 @@ module MU
285
299
  # Wrapper class for temporary Exceptions. Gives our internals something to
286
300
  # inherit that will log a notice message appropriately before bubbling up.
287
301
  class MuNonFatal < StandardError
288
- def initialize(message = nil)
289
- MU.log message, MU::NOTICE if !message.nil?
302
+ def initialize(message = nil, silent: false, details: nil)
303
+ MU.log message, MU::NOTICE, details: details if !message.nil? and !silent
290
304
  if MU.verbosity == MU::Logger::SILENT
291
305
  super ""
292
306
  else
@@ -295,6 +309,68 @@ module MU
295
309
  end
296
310
  end
297
311
 
312
+ # Boilerplate retry block executor, for making cloud API calls which might
313
+ # fail transiently.
314
+ #
315
+ # @param catchme [Array<Exception>]: Exception classes which should be caught and retried
316
+ # @param wait [Integer]: Number of seconds to wait between retries
317
+ # @param max [Integer]: Maximum number of retries; if less than 1, will retry indefinitely
318
+ # @param ignoreme [Array<Exception>]: Exception classes which can be silently treated as success. This will override any +loop_if+ block and return automatically (after invoking +always+, if the latter was specified).
319
+ # @param on_retry [Proc]: Optional block of code to invoke during retries
320
+ # @param always [Proc]: Optional block of code to invoke before returning or failing, a bit like +ensure+
321
+ # @param loop_if [Proc]: Optional block of code to invoke which will cause our block to be rerun until true
322
+ # @param loop_msg [String]: Message to display every third attempt
323
+ def self.retrier(catchme = nil, wait: 30, max: 0, ignoreme: [], on_retry: nil, always: nil, loop_if: nil, loop_msg: nil)
324
+
325
+ loop_if ||= Proc.new { false }
326
+
327
+ retries = 0
328
+ begin
329
+ retries += 1
330
+ loglevel = ((retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
331
+ log_attempts = retries.to_s
332
+ log_attempts += (max > 0 ? "/"+max.to_s : "")
333
+ yield(retries, wait) if block_given?
334
+ if loop_if.call
335
+ MU.log loop_msg, loglevel, details: log_attempts if loop_msg
336
+ sleep wait
337
+ end
338
+ rescue StandardError => e
339
+ if catchme and catchme.include?(e.class)
340
+ if max > 0 and retries >= max
341
+ always.call if always and always.is_a?(Proc)
342
+ if ignoreme.include?(e.class)
343
+ return
344
+ else
345
+ raise e
346
+ end
347
+ end
348
+
349
+ if on_retry and on_retry.is_a?(Proc)
350
+ on_retry.call(e)
351
+ end
352
+
353
+ if retries == max-1
354
+ MU.log e.message, MU::WARN, details: caller
355
+ sleep wait # wait extra on the final attempt
356
+ else
357
+ MU.log e.message, loglevel, details: log_attempts
358
+ end
359
+
360
+ sleep wait
361
+ retry
362
+ elsif ignoreme and ignoreme.include?(e.class)
363
+ always.call if always and always.is_a?(Proc)
364
+ return
365
+ else
366
+ always.call if always and always.is_a?(Proc)
367
+ raise e
368
+ end
369
+ end while loop_if.call and (max < 1 or retries < max)
370
+
371
+ always.call if always and always.is_a?(Proc)
372
+ end
373
+
298
374
  if !ENV.has_key?("MU_LIBDIR") and ENV.has_key?("MU_INSTALLDIR")
299
375
  ENV['MU_LIBDIR'] = ENV['MU_INSTALLDIR']+"/lib"
300
376
  else
@@ -394,20 +470,20 @@ module MU
394
470
  @@global_var_semaphore = Mutex.new
395
471
 
396
472
  # Set one of our global per-thread variables.
397
- def self.setVar(name, value)
473
+ def self.setVar(name, value, target_thread: Thread.current)
398
474
  @@global_var_semaphore.synchronize {
399
- @@globals[Thread.current.object_id] ||= Hash.new
400
- @@globals[Thread.current.object_id][name] ||= Hash.new
401
- @@globals[Thread.current.object_id][name] = value
475
+ @@globals[target_thread.object_id] ||= Hash.new
476
+ @@globals[target_thread.object_id][name] ||= Hash.new
477
+ @@globals[target_thread.object_id][name] = value
402
478
  }
403
479
  end
404
480
 
405
481
  # Copy the set of global variables in use by another thread, typically our
406
482
  # parent thread.
407
- def self.dupGlobals(parent_thread_id)
483
+ def self.dupGlobals(parent_thread_id, target_thread: Thread.current)
408
484
  @@globals[parent_thread_id] ||= {}
409
485
  @@globals[parent_thread_id].each_pair { |name, value|
410
- setVar(name, value)
486
+ setVar(name, value, target_thread: target_thread)
411
487
  }
412
488
  end
413
489
 
@@ -535,9 +611,10 @@ module MU
535
611
  end
536
612
 
537
613
  # Shortcut to invoke {MU::Logger#log}
538
- def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: nil, color: true)
614
+ def self.log(msg, level = MU::INFO, shorthand_details = nil, details: nil, html: false, verbosity: nil, color: true)
539
615
  return if (level == MU::DEBUG and verbosity and verbosity <= MU::Logger::LOUD)
540
616
  return if verbosity and verbosity == MU::Logger::SILENT
617
+ details ||= shorthand_details
541
618
 
542
619
  if (level == MU::ERR or
543
620
  level == MU::WARN or
@@ -556,25 +633,6 @@ module MU
556
633
  @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
557
634
  end
558
635
 
559
- # For log entries that should only be logged when we're in verbose mode
560
- DEBUG = 0.freeze
561
- # For ordinary log entries
562
- INFO = 1.freeze
563
- # For more interesting log entries which are not errors
564
- NOTICE = 2.freeze
565
- # Log entries for non-fatal errors
566
- WARN = 3.freeze
567
- # Log entries for non-fatal errors
568
- WARNING = 3.freeze
569
- # Log entries for fatal errors
570
- ERR = 4.freeze
571
- # Log entries for fatal errors
572
- ERROR = 4.freeze
573
- # Log entries that will be held and displayed/emailed at the end of deploy,
574
- # cleanup, etc.
575
- SUMMARY = 5.freeze
576
-
577
-
578
636
  autoload :Cleanup, 'mu/cleanup'
579
637
  autoload :Deploy, 'mu/deploy'
580
638
  autoload :MommaCat, 'mu/mommacat'
@@ -588,7 +646,7 @@ module MU
588
646
  new_cfg = $MU_CFG.dup
589
647
  examples = {}
590
648
  MU::Cloud.supportedClouds.each { |cloud|
591
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
649
+ cloudclass = MU::Cloud.cloudClass(cloud)
592
650
  begin
593
651
  if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
594
652
  cfg_blob = cloudclass.hosted_config
@@ -744,11 +802,7 @@ module MU
744
802
  # @param groomer [String]: The grooming agent to load.
745
803
  # @return [Class]: The class object implementing this groomer agent
746
804
  def self.loadGroomer(groomer)
747
- if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
748
- raise MuError, "Requested to use unsupported grooming agent #{groomer}"
749
- end
750
- require "mu/groomers/#{groomer.downcase}"
751
- return Object.const_get("MU").const_get("Groomer").const_get(groomer)
805
+ MU::Groomer.loadGroomer(groomer)
752
806
  end
753
807
 
754
808
  @@myRegion_var = nil
@@ -902,8 +956,7 @@ module MU
902
956
 
903
957
  @@myCloudDescriptor = nil
904
958
  if MU.myCloud
905
- svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
906
- found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
959
+ found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
907
960
  # found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
908
961
  if !found.nil? and found.size == 1
909
962
  @@myCloudDescriptor = found.values.first
@@ -916,8 +969,7 @@ module MU
916
969
  def self.myVPCObj
917
970
  return nil if MU.myCloud.nil?
918
971
  return @@myVPCObj_var if @@myVPCObj_var
919
- cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
920
- @@myVPCObj_var ||= cloudclass.myVPCObj
972
+ @@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
921
973
  @@myVPCObj_var
922
974
  end
923
975
 
@@ -1015,15 +1067,15 @@ module MU
1015
1067
 
1016
1068
  # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
1017
1069
  # return [String]: A password string.
1018
- def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 25)
1070
+ def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 50)
1019
1071
  # We have dopey complexity requirements, be stringent here.
1020
1072
  # I'll be nice and not condense this into one elegant-but-unreadable regular expression
1021
1073
  attempts = 0
1022
1074
  safe_metachars = Regexp.escape(safe_pattern)
1023
1075
  begin
1024
1076
  if attempts > retries
1025
- MU.log "Failed to generate an adequate Windows password after #{attempts}", MU::ERR
1026
- raise MuError, "Failed to generate an adequate Windows password after #{attempts}"
1077
+ MU.log "Failed to generate an adequate Windows password after #{attempts} attempts", MU::ERR
1078
+ raise MuError, "Failed to generate an adequate Windows password after #{attempts} attempts"
1027
1079
  end
1028
1080
  winpass = Password.random(14..16)
1029
1081
  attempts += 1
@@ -1042,10 +1094,9 @@ module MU
1042
1094
 
1043
1095
  clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
1044
1096
  clouds.each { |cloud|
1045
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1046
- bucketname = cloudclass.adminBucketName(credentials)
1097
+ bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
1047
1098
  begin
1048
- if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
1099
+ if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
1049
1100
  return bucketname
1050
1101
  end
1051
1102
  end