cloud-mu 3.1.4 → 3.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +16 -12
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -1
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +37 -12
  23. data/cloud-mu.gemspec +5 -3
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  27. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  28. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  29. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  30. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  31. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  32. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  33. data/extras/clean-stock-amis +25 -19
  34. data/extras/generate-stock-images +1 -0
  35. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  36. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  37. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  38. data/modules/mommacat.ru +1 -1
  39. data/modules/mu.rb +158 -107
  40. data/modules/mu/adoption.rb +386 -59
  41. data/modules/mu/cleanup.rb +214 -303
  42. data/modules/mu/cloud.rb +128 -1632
  43. data/modules/mu/cloud/database.rb +49 -0
  44. data/modules/mu/cloud/dnszone.rb +44 -0
  45. data/modules/mu/cloud/machine_images.rb +212 -0
  46. data/modules/mu/cloud/providers.rb +81 -0
  47. data/modules/mu/cloud/resource_base.rb +926 -0
  48. data/modules/mu/cloud/server.rb +40 -0
  49. data/modules/mu/cloud/server_pool.rb +1 -0
  50. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  51. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  52. data/modules/mu/cloud/wrappers.rb +169 -0
  53. data/modules/mu/config.rb +135 -82
  54. data/modules/mu/config/alarm.rb +2 -6
  55. data/modules/mu/config/bucket.rb +32 -3
  56. data/modules/mu/config/cache_cluster.rb +2 -2
  57. data/modules/mu/config/cdn.rb +100 -0
  58. data/modules/mu/config/collection.rb +1 -1
  59. data/modules/mu/config/container_cluster.rb +7 -2
  60. data/modules/mu/config/database.rb +84 -105
  61. data/modules/mu/config/database.yml +1 -2
  62. data/modules/mu/config/dnszone.rb +5 -4
  63. data/modules/mu/config/doc_helpers.rb +5 -6
  64. data/modules/mu/config/endpoint.rb +2 -1
  65. data/modules/mu/config/firewall_rule.rb +3 -19
  66. data/modules/mu/config/folder.rb +1 -1
  67. data/modules/mu/config/function.rb +17 -8
  68. data/modules/mu/config/group.rb +1 -1
  69. data/modules/mu/config/habitat.rb +1 -1
  70. data/modules/mu/config/job.rb +89 -0
  71. data/modules/mu/config/loadbalancer.rb +57 -11
  72. data/modules/mu/config/log.rb +1 -1
  73. data/modules/mu/config/msg_queue.rb +1 -1
  74. data/modules/mu/config/nosqldb.rb +1 -1
  75. data/modules/mu/config/notifier.rb +8 -19
  76. data/modules/mu/config/ref.rb +92 -14
  77. data/modules/mu/config/role.rb +1 -1
  78. data/modules/mu/config/schema_helpers.rb +38 -37
  79. data/modules/mu/config/search_domain.rb +1 -1
  80. data/modules/mu/config/server.rb +12 -13
  81. data/modules/mu/config/server.yml +1 -0
  82. data/modules/mu/config/server_pool.rb +3 -7
  83. data/modules/mu/config/storage_pool.rb +1 -1
  84. data/modules/mu/config/tail.rb +11 -0
  85. data/modules/mu/config/user.rb +1 -1
  86. data/modules/mu/config/vpc.rb +27 -23
  87. data/modules/mu/config/vpc.yml +0 -1
  88. data/modules/mu/defaults/AWS.yaml +91 -68
  89. data/modules/mu/defaults/Azure.yaml +1 -0
  90. data/modules/mu/defaults/Google.yaml +1 -0
  91. data/modules/mu/deploy.rb +33 -19
  92. data/modules/mu/groomer.rb +16 -1
  93. data/modules/mu/groomers/ansible.rb +123 -21
  94. data/modules/mu/groomers/chef.rb +64 -11
  95. data/modules/mu/logger.rb +120 -144
  96. data/modules/mu/master.rb +97 -4
  97. data/modules/mu/master/ssl.rb +0 -1
  98. data/modules/mu/mommacat.rb +154 -867
  99. data/modules/mu/mommacat/daemon.rb +23 -14
  100. data/modules/mu/mommacat/naming.rb +110 -3
  101. data/modules/mu/mommacat/search.rb +495 -0
  102. data/modules/mu/mommacat/storage.rb +225 -192
  103. data/modules/mu/{clouds → providers}/README.md +1 -1
  104. data/modules/mu/{clouds → providers}/aws.rb +281 -64
  105. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  106. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  107. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  108. data/modules/mu/providers/aws/cdn.rb +782 -0
  109. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  110. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
  111. data/modules/mu/providers/aws/database.rb +1744 -0
  112. data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
  113. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  114. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
  115. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  116. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  117. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  118. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  119. data/modules/mu/providers/aws/job.rb +466 -0
  120. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
  121. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  122. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  123. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  124. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  125. data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
  126. data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
  127. data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
  128. data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
  129. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  130. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  131. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  132. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  133. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  134. data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
  135. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  136. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  137. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  138. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  139. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  140. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  141. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  142. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  143. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  144. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  145. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  146. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  147. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  148. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  149. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  150. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  151. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  152. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  153. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  156. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  158. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  159. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  160. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  161. data/modules/mu/{clouds → providers}/google.rb +29 -6
  162. data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
  163. data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
  164. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  165. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  166. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  167. data/modules/mu/{clouds → providers}/google/function.rb +14 -8
  168. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  169. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  170. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  171. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  172. data/modules/mu/{clouds → providers}/google/server.rb +142 -55
  173. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  174. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  175. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  176. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  177. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  178. data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
  179. data/modules/tests/aws-jobs-functions.yaml +46 -0
  180. data/modules/tests/centos6.yaml +15 -0
  181. data/modules/tests/centos7.yaml +15 -0
  182. data/modules/tests/centos8.yaml +12 -0
  183. data/modules/tests/ecs.yaml +23 -0
  184. data/modules/tests/eks.yaml +1 -1
  185. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  186. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/microservice_app.yaml +288 -0
  189. data/modules/tests/rds.yaml +108 -0
  190. data/modules/tests/regrooms/rds.yaml +123 -0
  191. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  192. data/modules/tests/super_complex_bok.yml +2 -2
  193. data/modules/tests/super_simple_bok.yml +3 -5
  194. data/modules/tests/win2k12.yaml +25 -0
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +169 -93
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1974
  202. data/modules/mu/clouds/aws/endpoint.rb +0 -596
  203. data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -195,48 +195,49 @@ action :config do
195
195
  else
196
196
  # We want to run ec2config as admin user so Windows userdata executes as admin, however the local admin account doesn't have Logon As a Service right. Domain privileges are set separately
197
197
 
198
- cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
199
- source "ntrights"
200
- end
201
- [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
202
- pass = if usr == new_resource.ec2config_user
203
- new_resource.ec2config_password
204
- elsif usr == new_resource.ssh_user
205
- new_resource.ssh_password
206
- end
207
-
208
- user usr do
209
- password pass
210
- end
211
-
212
- group "Administrators" do
213
- action :modify
214
- members usr
215
- append true
216
- end
217
-
218
- %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
219
- batch "Grant local user #{usr} logon as service right" do
220
- code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
221
- end
222
- }
223
-
224
- # XXX user resource seems not to really be setting password, or is setting # in such a way that the user is being required to change it. Workaround.
225
- powershell_script "Adjust local account params for #{usr}" do
226
- code <<-EOH
227
- (([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
228
- EOH
229
- end
230
-
231
- if usr == new_resource.ssh_user
232
-
233
- %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
234
- batch "Grant local user #{usr} logon as service right" do
235
- code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
236
- end
237
- }
238
-
239
- end
240
- }
198
+ # cookbook_file "c:\\Windows\\SysWOW64\\ntrights.exe" do
199
+ # source "ntrights"
200
+ # end
201
+ # [new_resource.ssh_user, new_resource.ec2config_user].each { |usr|
202
+ # pass = if usr == new_resource.ec2config_user
203
+ # new_resource.ec2config_password
204
+ # elsif usr == new_resource.ssh_user
205
+ # new_resource.ssh_password
206
+ # end
207
+ #
208
+ # user usr do
209
+ # password pass
210
+ # action :modify
211
+ # end
212
+ #
213
+ # group "Administrators" do
214
+ # action :modify
215
+ # members usr
216
+ # append true
217
+ # end
218
+ #
219
+ # %w{SeDenyRemoteInteractiveLogonRight SeDenyInteractiveLogonRight SeServiceLogonRight}.each { |privilege|
220
+ # batch "Grant local user #{usr} logon as service right" do
221
+ # code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
222
+ # end
223
+ # }
224
+ #
225
+ # # XXX user resource seems not to really be setting password, or is setting # in such a way that the user is being required to change it. Workaround.
226
+ # powershell_script "Adjust local account params for #{usr}" do
227
+ # code <<-EOH
228
+ # (([adsi]('WinNT://./#{usr}, user')).psbase.invoke('SetPassword', '#{pass}'))
229
+ # EOH
230
+ # end
231
+ #
232
+ # if usr == new_resource.ssh_user
233
+ #
234
+ # %w{SeCreateTokenPrivilege SeTcbPrivilege SeAssignPrimaryTokenPrivilege}.each { |privilege|
235
+ # batch "Grant local user #{usr} logon as service right" do
236
+ # code "C:\\Windows\\SysWOW64\\ntrights +r #{privilege} -u #{usr}"
237
+ # end
238
+ # }
239
+ #
240
+ # end
241
+ # }
241
242
  end
242
243
  end
@@ -18,37 +18,43 @@ require 'json'
18
18
  require File.realpath(File.expand_path(File.dirname(__FILE__)+"/../bin/mu-load-config.rb"))
19
19
  require 'mu'
20
20
 
21
- credentials = if ARGV[0] and !ARGV[0].empty?
22
- ARGV[0]
23
- else
24
- nil
21
+ $opts = Optimist::options do
22
+ banner <<-EOS
23
+ #{$0} [-c credentials] [-i imagename]
24
+ EOS
25
+ opt :credentials, "Use these AWS credentials from mu.yaml instead of the default set", :required => false, :type => :string
26
+ opt :image, "Purge a specific image, instead of just scrubing old ones", :required => false, :type => :string
25
27
  end
26
28
 
27
29
  filters = [
28
30
  {
29
31
  name: "owner-id",
30
- values: [MU::Cloud::AWS.credToAcct(credentials)]
32
+ values: [MU::Cloud::AWS.credToAcct($opts[:credentials])]
31
33
  }
32
34
  ]
33
35
 
34
36
 
35
37
  MU::Cloud::AWS.listRegions.each { | r|
36
- images = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_images(
38
+ images = MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).describe_images(
37
39
  filters: filters + [{ "name" => "state", "values" => ["available"]}]
38
40
  ).images
39
41
  images.each { |ami|
40
- if (DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/)
41
- snaps = []
42
- ami.block_device_mappings.each { |dev|
43
- if !dev.ebs.nil?
44
- snaps << dev.ebs.snapshot_id
45
- end
46
- }
47
- MU.log "Deregistering #{ami.name} (#{ami.creation_date})", MU::WARN, details: snaps
48
- MU::Cloud::AWS.ec2(region: r, credentials: credentials).deregister_image(image_id: ami.image_id)
49
- snaps.each { |snap_id|
50
- MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_snapshot(snapshot_id: snap_id)
51
- }
52
- end
42
+ if ($opts[:image] and ami.name == $opts[:image]) or
43
+ ((DateTime.now.to_time - DateTime.parse(ami.creation_date).to_time) > 15552000 and ami.name.match(/^MU-(PROD|DEV)/))
44
+ snaps = []
45
+ ami.block_device_mappings.each { |dev|
46
+ if !dev.ebs.nil?
47
+ snaps << dev.ebs.snapshot_id
48
+ end
49
+ }
50
+ MU.log "Deregistering #{ami.name}, #{r} (#{ami.creation_date})", MU::WARN, details: snaps
51
+ begin
52
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).deregister_image(image_id: ami.image_id)
53
+ rescue Aws::EC2::Errors::InvalidAMIIDUnavailable
54
+ end
55
+ snaps.each { |snap_id|
56
+ MU::Cloud::AWS.ec2(region: r, credentials: $opts[:credentials]).delete_snapshot(snapshot_id: snap_id)
57
+ }
58
+ end
53
59
  }
54
60
  }
@@ -91,6 +91,7 @@ $opts[:clouds].each { |cloud|
91
91
  end
92
92
  next if !needed
93
93
  end
94
+ MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
94
95
  conf_engine = MU::Config.new(
95
96
  bok_dir+"/"+cloud+"/"+platform+".yaml",
96
97
  default_credentials: $opts[(cloud.downcase+"_creds").to_sym]
@@ -1,16 +1,21 @@
1
1
  ---
2
2
  appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
3
5
  servers:
4
- -
5
- name: win2k12
6
- platform: windows
7
- size: m4.large
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::updates]
11
- - recipe[mu-utility::cleanup_image_helper]
12
- create_image:
13
- image_then_destroy: true
14
- public: true
15
- copy_to_regions:
16
- - "#ALL"
6
+ - name: win2k12
7
+ platform: win2k12
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -1,16 +1,21 @@
1
1
  ---
2
2
  appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
3
5
  servers:
4
- -
5
- name: win2k16
6
- platform: windows
7
- size: m4.large
8
- scrub_groomer: true
9
- run_list:
10
- - recipe[mu-tools::updates]
11
- - recipe[mu-utility::cleanup_image_helper]
12
- create_image:
13
- image_then_destroy: true
14
- public: true
15
- copy_to_regions:
16
- - "#ALL"
6
+ - name: win2k16
7
+ platform: win2k16
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -0,0 +1,21 @@
1
+ ---
2
+ appname: mu
3
+ vpcs:
4
+ - name: windowsbuild
5
+ servers:
6
+ - name: win2k19
7
+ platform: windows
8
+ vpc:
9
+ name: windowsbuild
10
+ size: m4.large
11
+ scrub_groomer: true
12
+ groomer: Ansible
13
+ run_list:
14
+ - mu-windows
15
+ ansible_vars:
16
+ mu_build_image: true
17
+ create_image:
18
+ image_then_destroy: true
19
+ public: true
20
+ copy_to_regions:
21
+ - "#ALL"
@@ -51,7 +51,7 @@ Signal.trap("URG") do
51
51
  end
52
52
 
53
53
  begin
54
- MU::MommaCat.syncMonitoringConfig(false)
54
+ MU::Master.syncMonitoringConfig(false)
55
55
  rescue StandardError => e
56
56
  MU.log e.inspect, MU::ERR, details: e.backtrace
57
57
  # ...but don't die!
@@ -79,38 +79,40 @@ class Hash
79
79
  }
80
80
  return 0 if self == other # that was easy!
81
81
  # compare elements and decide who's "bigger" based on their totals?
82
- 0
82
+
83
+ # fine, try some brute force and just hope everything implements to_s
84
+ self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
83
85
  end
84
86
 
85
- # Recursively compare two hashes
86
- def diff(with, on = self, level: 0, parents: [])
87
+ # Recursively compare two Mu Basket of Kittens hashes and report the differences
88
+ def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
87
89
  return if with.nil? and on.nil?
88
90
  if with.nil? or on.nil? or with.class != on.class
89
91
  return # XXX ...however we're flagging differences
90
92
  end
91
93
  return if on == with
92
94
 
93
- tree = ""
94
- indentsize = 0
95
- parents.each { |p|
96
- tree += (" " * indentsize) + p + " => \n"
97
- indentsize += 2
98
- }
99
- indent = (" " * indentsize)
100
-
101
95
  changes = []
96
+ report ||= {}
102
97
  if on.is_a?(Hash)
103
98
  on_unique = (on.keys - with.keys)
104
99
  with_unique = (with.keys - on.keys)
105
100
  shared = (with.keys & on.keys)
106
101
  shared.each { |k|
107
- diff(with[k], on[k], level: level+1, parents: parents + [k])
102
+
103
+ report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
104
+ if report_data and !report_data.empty?
105
+ report ||= {}
106
+ report[k] = report_data
107
+ end
108
108
  }
109
109
  on_unique.each { |k|
110
- changes << "- ".red+PP.pp({k => on[k] }, '')
110
+ report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
111
+ report[k][:habitat] = habitat if habitat
111
112
  }
112
113
  with_unique.each { |k|
113
- changes << "+ ".green+PP.pp({k => with[k]}, '')
114
+ report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
115
+ report[k][:habitat] = habitat if habitat
114
116
  }
115
117
  elsif on.is_a?(Array)
116
118
  return if with == on
@@ -122,29 +124,27 @@ class Hash
122
124
  # sorting arrays full of weird, non-primitive types.
123
125
  done = []
124
126
  on.sort.each { |elt|
125
- if elt.is_a?(Hash) and elt['name'] or elt['entity']# or elt['cloud_id']
126
- with.sort.each { |other_elt|
127
- # Figure out what convention this thing is using for resource identification
128
- compare_a, compare_b = if elt['name'].nil? and elt["id"].nil? and !elt["entity"].nil? and !other_elt["entity"].nil?
129
- [elt["entity"], other_elt["entity"]]
130
- else
131
- [elt, other_elt]
132
- end
127
+ if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
128
+ elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
133
129
 
134
- if (compare_a['name'] and compare_b['name'] == compare_a['name']) or
135
- (compare_a['name'].nil? and !compare_a["id"].nil? and compare_a["id"] == compare_b["id"])
136
- break if elt == other_elt
130
+ with.sort.each { |other_elt|
131
+ other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
132
+
133
+ # Case 1: The array element exists in both version of this array
134
+ if elt_namestr and other_elt_namestr and
135
+ elt_namestr == other_elt_namestr and
136
+ (elt_location.nil? or other_elt_location.nil? or
137
+ elt_location == other_elt_location or
138
+ !(elt_location_list & other_elt_location_list).empty?
139
+ )
137
140
  done << elt
138
141
  done << other_elt
139
- namestr = if elt['type']
140
- "#{elt['type']}[#{elt['name']}]"
141
- elsif elt['name']
142
- elt['name']
143
- elsif elt['entity'] and elt["entity"]["id"]
144
- elt['entity']['id']
142
+ break if elt == other_elt # if they're identical, we're done
143
+ report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
144
+ if report_data and !report_data.empty?
145
+ report ||= {}
146
+ report[elt_namestr] = report_data
145
147
  end
146
-
147
- diff(other_elt, elt, level: level+1, parents: parents + [namestr])
148
148
  break
149
149
  end
150
150
  }
@@ -152,43 +152,34 @@ class Hash
152
152
  }
153
153
  on_unique = (on - with) - done
154
154
  with_unique = (with - on) - done
155
- # if on_unique.size > 0 or with_unique.size > 0
156
- # if before_a != after_a
157
- # MU.log "A BEFORE", MU::NOTICE, details: before_a
158
- # MU.log "A AFTER", MU::NOTICE, details: after_a
159
- # end
160
- # if before_b != after_b
161
- # MU.log "B BEFORE", MU::NOTICE, details: before_b
162
- # MU.log "B AFTER", MU::NOTICE, details: after_b
163
- # end
164
- # end
155
+
156
+ # Case 2: This array entry exists in the old version, but not the new one
165
157
  on_unique.each { |e|
166
- changes << if e.is_a?(Hash)
167
- "- ".red+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n "+(indent))
168
- else
169
- "- ".red+e.to_s
170
- end
158
+ namestr, loc = MU::MommaCat.getChunkName(e)
159
+
160
+ report ||= {}
161
+ report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
162
+ report[namestr][:habitat] = loc if loc
171
163
  }
164
+
165
+ # Case 3: This array entry exists in the new version, but not the old one
172
166
  with_unique.each { |e|
173
- changes << if e.is_a?(Hash)
174
- "+ ".green+PP.pp(Hash.bok_minimize(e), '').gsub(/\n/, "\n "+(indent))
175
- else
176
- "+ ".green+e.to_s
177
- end
167
+ namestr, loc = MU::MommaCat.getChunkName(e)
168
+
169
+ report ||= {}
170
+ report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
171
+ report[namestr][:habitat] = loc if loc
178
172
  }
173
+
174
+ # A plain old leaf node of data
179
175
  else
180
176
  if on != with
181
- changes << "-".red+" #{on.to_s}"
182
- changes << "+".green+" #{with.to_s}"
177
+ report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
178
+ report[:habitat] = habitat if habitat
183
179
  end
184
180
  end
185
181
 
186
- if changes.size > 0
187
- puts tree
188
- changes.each { |c|
189
- puts indent+c
190
- }
191
- end
182
+ report.freeze
192
183
  end
193
184
 
194
185
  # Implement a merge! that just updates each hash leaf as needed, not
@@ -212,8 +203,29 @@ class Hash
212
203
  end
213
204
 
214
205
  ENV['HOME'] = Etc.getpwuid(Process.uid).dir
206
+ module MU
207
+
208
+ # For log entries that should only be logged when we're in verbose mode
209
+ DEBUG = 0.freeze
210
+ # For ordinary log entries
211
+ INFO = 1.freeze
212
+ # For more interesting log entries which are not errors
213
+ NOTICE = 2.freeze
214
+ # Log entries for non-fatal errors
215
+ WARN = 3.freeze
216
+ # Log entries for non-fatal errors
217
+ WARNING = 3.freeze
218
+ # Log entries for fatal errors
219
+ ERR = 4.freeze
220
+ # Log entries for fatal errors
221
+ ERROR = 4.freeze
222
+ # Log entries that will be held and displayed/emailed at the end of deploy,
223
+ # cleanup, etc.
224
+ SUMMARY = 5.freeze
225
+ end
215
226
 
216
227
  require 'mu/logger'
228
+
217
229
  module MU
218
230
 
219
231
  # Subclass core thread so we can gracefully handle it when we hit system
@@ -263,6 +275,7 @@ module MU
263
275
  end while newguy.nil?
264
276
 
265
277
  @@mu_global_thread_semaphore.synchronize {
278
+ MU.dupGlobals(Thread.current.object_id, target_thread: newguy)
266
279
  @@mu_global_threads << newguy
267
280
  }
268
281
 
@@ -272,8 +285,9 @@ module MU
272
285
  # Wrapper class for fatal Exceptions. Gives our internals something to
273
286
  # inherit that will log an error message appropriately before bubbling up.
274
287
  class MuError < StandardError
275
- def initialize(message = nil)
276
- MU.log message, MU::ERR, details: caller[2] if !message.nil?
288
+ def initialize(message = nil, silent: false, details: nil)
289
+ details ||= caller[2]
290
+ MU.log message, MU::ERR, details: details if !message.nil? and !silent
277
291
  if MU.verbosity == MU::Logger::SILENT
278
292
  super ""
279
293
  else
@@ -285,8 +299,8 @@ module MU
285
299
  # Wrapper class for temporary Exceptions. Gives our internals something to
286
300
  # inherit that will log a notice message appropriately before bubbling up.
287
301
  class MuNonFatal < StandardError
288
- def initialize(message = nil)
289
- MU.log message, MU::NOTICE if !message.nil?
302
+ def initialize(message = nil, silent: false, details: nil)
303
+ MU.log message, MU::NOTICE, details: details if !message.nil? and !silent
290
304
  if MU.verbosity == MU::Logger::SILENT
291
305
  super ""
292
306
  else
@@ -295,6 +309,68 @@ module MU
295
309
  end
296
310
  end
297
311
 
312
+ # Boilerplate retry block executor, for making cloud API calls which might
313
+ # fail transiently.
314
+ #
315
+ # @param catchme [Array<Exception>]: Exception classes which should be caught and retried
316
+ # @param wait [Integer]: Number of seconds to wait between retries
317
+ # @param max [Integer]: Maximum number of retries; if less than 1, will retry indefinitely
318
+ # @param ignoreme [Array<Exception>]: Exception classes which can be silently treated as success. This will override any +loop_if+ block and return automatically (after invoking +always+, if the latter was specified).
319
+ # @param on_retry [Proc]: Optional block of code to invoke during retries
320
+ # @param always [Proc]: Optional block of code to invoke before returning or failing, a bit like +ensure+
321
+ # @param loop_if [Proc]: Optional block of code to invoke which will cause our block to be rerun until true
322
+ # @param loop_msg [String]: Message to display every third attempt
323
+ def self.retrier(catchme = nil, wait: 30, max: 0, ignoreme: [], on_retry: nil, always: nil, loop_if: nil, loop_msg: nil)
324
+
325
+ loop_if ||= Proc.new { false }
326
+
327
+ retries = 0
328
+ begin
329
+ retries += 1
330
+ loglevel = ((retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
331
+ log_attempts = retries.to_s
332
+ log_attempts += (max > 0 ? "/"+max.to_s : "")
333
+ yield(retries, wait) if block_given?
334
+ if loop_if.call
335
+ MU.log loop_msg, loglevel, details: log_attempts if loop_msg
336
+ sleep wait
337
+ end
338
+ rescue StandardError => e
339
+ if catchme and catchme.include?(e.class)
340
+ if max > 0 and retries >= max
341
+ always.call if always and always.is_a?(Proc)
342
+ if ignoreme.include?(e.class)
343
+ return
344
+ else
345
+ raise e
346
+ end
347
+ end
348
+
349
+ if on_retry and on_retry.is_a?(Proc)
350
+ on_retry.call(e)
351
+ end
352
+
353
+ if retries == max-1
354
+ MU.log e.message, MU::WARN, details: caller
355
+ sleep wait # wait extra on the final attempt
356
+ else
357
+ MU.log e.message, loglevel, details: log_attempts
358
+ end
359
+
360
+ sleep wait
361
+ retry
362
+ elsif ignoreme and ignoreme.include?(e.class)
363
+ always.call if always and always.is_a?(Proc)
364
+ return
365
+ else
366
+ always.call if always and always.is_a?(Proc)
367
+ raise e
368
+ end
369
+ end while loop_if.call and (max < 1 or retries < max)
370
+
371
+ always.call if always and always.is_a?(Proc)
372
+ end
373
+
298
374
  if !ENV.has_key?("MU_LIBDIR") and ENV.has_key?("MU_INSTALLDIR")
299
375
  ENV['MU_LIBDIR'] = ENV['MU_INSTALLDIR']+"/lib"
300
376
  else
@@ -394,20 +470,20 @@ module MU
394
470
  @@global_var_semaphore = Mutex.new
395
471
 
396
472
  # Set one of our global per-thread variables.
397
- def self.setVar(name, value)
473
+ def self.setVar(name, value, target_thread: Thread.current)
398
474
  @@global_var_semaphore.synchronize {
399
- @@globals[Thread.current.object_id] ||= Hash.new
400
- @@globals[Thread.current.object_id][name] ||= Hash.new
401
- @@globals[Thread.current.object_id][name] = value
475
+ @@globals[target_thread.object_id] ||= Hash.new
476
+ @@globals[target_thread.object_id][name] ||= Hash.new
477
+ @@globals[target_thread.object_id][name] = value
402
478
  }
403
479
  end
404
480
 
405
481
  # Copy the set of global variables in use by another thread, typically our
406
482
  # parent thread.
407
- def self.dupGlobals(parent_thread_id)
483
+ def self.dupGlobals(parent_thread_id, target_thread: Thread.current)
408
484
  @@globals[parent_thread_id] ||= {}
409
485
  @@globals[parent_thread_id].each_pair { |name, value|
410
- setVar(name, value)
486
+ setVar(name, value, target_thread: target_thread)
411
487
  }
412
488
  end
413
489
 
@@ -535,9 +611,10 @@ module MU
535
611
  end
536
612
 
537
613
  # Shortcut to invoke {MU::Logger#log}
538
- def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: nil, color: true)
614
+ def self.log(msg, level = MU::INFO, shorthand_details = nil, details: nil, html: false, verbosity: nil, color: true)
539
615
  return if (level == MU::DEBUG and verbosity and verbosity <= MU::Logger::LOUD)
540
616
  return if verbosity and verbosity == MU::Logger::SILENT
617
+ details ||= shorthand_details
541
618
 
542
619
  if (level == MU::ERR or
543
620
  level == MU::WARN or
@@ -556,25 +633,6 @@ module MU
556
633
  @@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
557
634
  end
558
635
 
559
- # For log entries that should only be logged when we're in verbose mode
560
- DEBUG = 0.freeze
561
- # For ordinary log entries
562
- INFO = 1.freeze
563
- # For more interesting log entries which are not errors
564
- NOTICE = 2.freeze
565
- # Log entries for non-fatal errors
566
- WARN = 3.freeze
567
- # Log entries for non-fatal errors
568
- WARNING = 3.freeze
569
- # Log entries for fatal errors
570
- ERR = 4.freeze
571
- # Log entries for fatal errors
572
- ERROR = 4.freeze
573
- # Log entries that will be held and displayed/emailed at the end of deploy,
574
- # cleanup, etc.
575
- SUMMARY = 5.freeze
576
-
577
-
578
636
  autoload :Cleanup, 'mu/cleanup'
579
637
  autoload :Deploy, 'mu/deploy'
580
638
  autoload :MommaCat, 'mu/mommacat'
@@ -588,7 +646,7 @@ module MU
588
646
  new_cfg = $MU_CFG.dup
589
647
  examples = {}
590
648
  MU::Cloud.supportedClouds.each { |cloud|
591
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
649
+ cloudclass = MU::Cloud.cloudClass(cloud)
592
650
  begin
593
651
  if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
594
652
  cfg_blob = cloudclass.hosted_config
@@ -744,11 +802,7 @@ module MU
744
802
  # @param groomer [String]: The grooming agent to load.
745
803
  # @return [Class]: The class object implementing this groomer agent
746
804
  def self.loadGroomer(groomer)
747
- if !File.size?(MU.myRoot+"/modules/mu/groomers/#{groomer.downcase}.rb")
748
- raise MuError, "Requested to use unsupported grooming agent #{groomer}"
749
- end
750
- require "mu/groomers/#{groomer.downcase}"
751
- return Object.const_get("MU").const_get("Groomer").const_get(groomer)
805
+ MU::Groomer.loadGroomer(groomer)
752
806
  end
753
807
 
754
808
  @@myRegion_var = nil
@@ -902,8 +956,7 @@ module MU
902
956
 
903
957
  @@myCloudDescriptor = nil
904
958
  if MU.myCloud
905
- svrclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud).const_get("Server")
906
- found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
959
+ found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
907
960
  # found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
908
961
  if !found.nil? and found.size == 1
909
962
  @@myCloudDescriptor = found.values.first
@@ -916,8 +969,7 @@ module MU
916
969
  def self.myVPCObj
917
970
  return nil if MU.myCloud.nil?
918
971
  return @@myVPCObj_var if @@myVPCObj_var
919
- cloudclass = const_get("MU").const_get("Cloud").const_get(MU.myCloud)
920
- @@myVPCObj_var ||= cloudclass.myVPCObj
972
+ @@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
921
973
  @@myVPCObj_var
922
974
  end
923
975
 
@@ -1015,15 +1067,15 @@ module MU
1015
1067
 
1016
1068
  # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
1017
1069
  # return [String]: A password string.
1018
- def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 25)
1070
+ def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 50)
1019
1071
  # We have dopey complexity requirements, be stringent here.
1020
1072
  # I'll be nice and not condense this into one elegant-but-unreadable regular expression
1021
1073
  attempts = 0
1022
1074
  safe_metachars = Regexp.escape(safe_pattern)
1023
1075
  begin
1024
1076
  if attempts > retries
1025
- MU.log "Failed to generate an adequate Windows password after #{attempts}", MU::ERR
1026
- raise MuError, "Failed to generate an adequate Windows password after #{attempts}"
1077
+ MU.log "Failed to generate an adequate Windows password after #{attempts} attempts", MU::ERR
1078
+ raise MuError, "Failed to generate an adequate Windows password after #{attempts} attempts"
1027
1079
  end
1028
1080
  winpass = Password.random(14..16)
1029
1081
  attempts += 1
@@ -1042,10 +1094,9 @@ module MU
1042
1094
 
1043
1095
  clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
1044
1096
  clouds.each { |cloud|
1045
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1046
- bucketname = cloudclass.adminBucketName(credentials)
1097
+ bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
1047
1098
  begin
1048
- if platform or (cloudclass.hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
1099
+ if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
1049
1100
  return bucketname
1050
1101
  end
1051
1102
  end