cloud-mu 3.1.6 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. checksums.yaml +4 -4
  2. data/bin/mu-adopt +4 -12
  3. data/bin/mu-azure-tests +57 -0
  4. data/bin/mu-cleanup +2 -4
  5. data/bin/mu-configure +37 -1
  6. data/bin/mu-deploy +3 -3
  7. data/bin/mu-findstray-tests +25 -0
  8. data/bin/mu-gen-docs +2 -4
  9. data/bin/mu-run-tests +23 -10
  10. data/cloud-mu.gemspec +2 -2
  11. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  12. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  13. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  14. data/extras/generate-stock-images +1 -0
  15. data/modules/mu.rb +82 -95
  16. data/modules/mu/adoption.rb +356 -56
  17. data/modules/mu/cleanup.rb +21 -20
  18. data/modules/mu/cloud.rb +79 -1753
  19. data/modules/mu/cloud/database.rb +49 -0
  20. data/modules/mu/cloud/dnszone.rb +46 -0
  21. data/modules/mu/cloud/machine_images.rb +212 -0
  22. data/modules/mu/cloud/providers.rb +81 -0
  23. data/modules/mu/cloud/resource_base.rb +920 -0
  24. data/modules/mu/cloud/server.rb +40 -0
  25. data/modules/mu/cloud/server_pool.rb +1 -0
  26. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  27. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  28. data/modules/mu/cloud/wrappers.rb +165 -0
  29. data/modules/mu/config.rb +122 -80
  30. data/modules/mu/config/alarm.rb +2 -6
  31. data/modules/mu/config/bucket.rb +1 -1
  32. data/modules/mu/config/cache_cluster.rb +1 -1
  33. data/modules/mu/config/collection.rb +1 -1
  34. data/modules/mu/config/container_cluster.rb +2 -2
  35. data/modules/mu/config/database.rb +83 -104
  36. data/modules/mu/config/database.yml +1 -2
  37. data/modules/mu/config/dnszone.rb +1 -1
  38. data/modules/mu/config/doc_helpers.rb +4 -5
  39. data/modules/mu/config/endpoint.rb +1 -1
  40. data/modules/mu/config/firewall_rule.rb +3 -19
  41. data/modules/mu/config/folder.rb +1 -1
  42. data/modules/mu/config/function.rb +1 -1
  43. data/modules/mu/config/group.rb +1 -1
  44. data/modules/mu/config/habitat.rb +1 -1
  45. data/modules/mu/config/loadbalancer.rb +57 -11
  46. data/modules/mu/config/log.rb +1 -1
  47. data/modules/mu/config/msg_queue.rb +1 -1
  48. data/modules/mu/config/nosqldb.rb +1 -1
  49. data/modules/mu/config/notifier.rb +1 -1
  50. data/modules/mu/config/ref.rb +30 -4
  51. data/modules/mu/config/role.rb +1 -1
  52. data/modules/mu/config/schema_helpers.rb +30 -34
  53. data/modules/mu/config/search_domain.rb +1 -1
  54. data/modules/mu/config/server.rb +4 -12
  55. data/modules/mu/config/server_pool.rb +3 -7
  56. data/modules/mu/config/storage_pool.rb +1 -1
  57. data/modules/mu/config/tail.rb +10 -0
  58. data/modules/mu/config/user.rb +1 -1
  59. data/modules/mu/config/vpc.rb +12 -17
  60. data/modules/mu/defaults/AWS.yaml +32 -32
  61. data/modules/mu/defaults/Azure.yaml +1 -0
  62. data/modules/mu/defaults/Google.yaml +1 -0
  63. data/modules/mu/deploy.rb +16 -15
  64. data/modules/mu/groomer.rb +15 -0
  65. data/modules/mu/groomers/chef.rb +3 -0
  66. data/modules/mu/logger.rb +120 -144
  67. data/modules/mu/master.rb +1 -1
  68. data/modules/mu/mommacat.rb +54 -25
  69. data/modules/mu/mommacat/daemon.rb +10 -7
  70. data/modules/mu/mommacat/naming.rb +82 -3
  71. data/modules/mu/mommacat/search.rb +47 -15
  72. data/modules/mu/mommacat/storage.rb +72 -41
  73. data/modules/mu/{clouds → providers}/README.md +1 -1
  74. data/modules/mu/{clouds → providers}/aws.rb +114 -47
  75. data/modules/mu/{clouds → providers}/aws/alarm.rb +1 -1
  76. data/modules/mu/{clouds → providers}/aws/bucket.rb +2 -2
  77. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +10 -46
  78. data/modules/mu/{clouds → providers}/aws/collection.rb +3 -3
  79. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +15 -33
  80. data/modules/mu/providers/aws/database.rb +1744 -0
  81. data/modules/mu/{clouds → providers}/aws/dnszone.rb +2 -5
  82. data/modules/mu/{clouds → providers}/aws/endpoint.rb +2 -11
  83. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +33 -29
  84. data/modules/mu/{clouds → providers}/aws/folder.rb +0 -0
  85. data/modules/mu/{clouds → providers}/aws/function.rb +2 -10
  86. data/modules/mu/{clouds → providers}/aws/group.rb +9 -13
  87. data/modules/mu/{clouds → providers}/aws/habitat.rb +1 -1
  88. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +41 -33
  89. data/modules/mu/{clouds → providers}/aws/log.rb +2 -2
  90. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +2 -8
  91. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +0 -0
  92. data/modules/mu/{clouds → providers}/aws/notifier.rb +0 -0
  93. data/modules/mu/{clouds → providers}/aws/role.rb +7 -7
  94. data/modules/mu/{clouds → providers}/aws/search_domain.rb +8 -13
  95. data/modules/mu/{clouds → providers}/aws/server.rb +55 -90
  96. data/modules/mu/{clouds → providers}/aws/server_pool.rb +10 -33
  97. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +19 -36
  98. data/modules/mu/{clouds → providers}/aws/user.rb +8 -12
  99. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  100. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  101. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  102. data/modules/mu/{clouds → providers}/aws/vpc.rb +135 -70
  103. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  104. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  105. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  106. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  107. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  108. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  109. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  110. data/modules/mu/{clouds → providers}/azure/server.rb +30 -23
  111. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  112. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  113. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  114. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  115. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  116. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  117. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  118. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  119. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  120. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  121. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  122. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  123. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  124. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  125. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  126. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  127. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  128. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  129. data/modules/mu/{clouds → providers}/google.rb +14 -6
  130. data/modules/mu/{clouds → providers}/google/bucket.rb +1 -1
  131. data/modules/mu/{clouds → providers}/google/container_cluster.rb +28 -13
  132. data/modules/mu/{clouds → providers}/google/database.rb +1 -8
  133. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +2 -2
  134. data/modules/mu/{clouds → providers}/google/folder.rb +4 -8
  135. data/modules/mu/{clouds → providers}/google/function.rb +3 -3
  136. data/modules/mu/{clouds → providers}/google/group.rb +8 -16
  137. data/modules/mu/{clouds → providers}/google/habitat.rb +3 -7
  138. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +1 -1
  139. data/modules/mu/{clouds → providers}/google/role.rb +42 -34
  140. data/modules/mu/{clouds → providers}/google/server.rb +25 -10
  141. data/modules/mu/{clouds → providers}/google/server_pool.rb +10 -10
  142. data/modules/mu/{clouds → providers}/google/user.rb +31 -21
  143. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  144. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  145. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  146. data/modules/mu/{clouds → providers}/google/vpc.rb +37 -2
  147. data/modules/tests/centos6.yaml +11 -0
  148. data/modules/tests/centos7.yaml +11 -0
  149. data/modules/tests/centos8.yaml +12 -0
  150. data/modules/tests/rds.yaml +108 -0
  151. data/modules/tests/regrooms/rds.yaml +123 -0
  152. data/spec/mu/clouds/azure_spec.rb +2 -2
  153. metadata +108 -89
  154. data/modules/mu/clouds/aws/database.rb +0 -1974
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/storage_pool.rb
18
18
  class StoragePool
19
19
 
20
20
  # Base configuration schema for a StoragePool
@@ -117,6 +117,16 @@ module MU
117
117
  def gsub(*args)
118
118
  to_s.gsub(*args)
119
119
  end
120
+
121
+ # Lets callers access us like a {Hash}
122
+ # @param attribute [String,Symbol]
123
+ def [](attribute)
124
+ if respond_to?(attribute.to_sym)
125
+ send(attribute.to_sym)
126
+ else
127
+ nil
128
+ end
129
+ end
120
130
  end
121
131
 
122
132
  # Wrapper method for creating a {MU::Config::Tail} object as a reference to
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/user.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/user.rb
18
18
  class User
19
19
 
20
20
  # Base configuration schema for a User
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/vpc.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/vpc.rb
18
18
  class VPC
19
19
 
20
20
  # Base configuration schema for a VPC
@@ -419,7 +419,7 @@ module MU
419
419
  if configurator.updating and configurator.existing_deploy and
420
420
  configurator.existing_deploy.original_config['vpcs']
421
421
  configurator.existing_deploy.original_config['vpcs'].each { |v|
422
- if v['name'] == vpc['name']
422
+ if v['name'].to_s == vpc['name'].to_s
423
423
  vpc['ip_block'] = v['ip_block']
424
424
  vpc['peers'] ||= []
425
425
  vpc['peers'].concat(v['peers'])
@@ -431,6 +431,10 @@ module MU
431
431
  break
432
432
  end
433
433
  }
434
+ if !vpc['ip_block']
435
+ MU.log "Loading existing deploy but can't find IP block of VPC #{vpc['name']}", MU::ERR
436
+ ok = false
437
+ end
434
438
  else
435
439
  using_default_cidr = true
436
440
  vpc['ip_block'] = "10.0.0.0/16"
@@ -539,7 +543,7 @@ module MU
539
543
  # Clouds that don't have some kind of native NAT gateway can also
540
544
  # leverage this host to honor "gateway" => "#NAT" situations.
541
545
  if !can_peer and !already_peered and have_public and vpc["create_bastion"]
542
- serverclass = Object.const_get("MU").const_get("Cloud").const_get(vpc["cloud"]).const_get("Server")
546
+ serverclass = MU::Cloud.resourceClass(vpc["cloud"], "Server")
543
547
  bastion = serverclass.genericNAT.dup
544
548
  bastion["groomer_variables"] = {
545
549
  "nat_ip_block" => vpc["ip_block"].to_s
@@ -558,10 +562,7 @@ module MU
558
562
  "name" => vpc["name"],
559
563
  "subnet_pref" => "public"
560
564
  }
561
- vpc["dependencies"] << {
562
- "type" => "server",
563
- "name" => bastion['name'],
564
- }
565
+ MU::Config.addDependency(vpc, bastion['name'], "server", no_create_wait: true)
565
566
  vpc["bastion"] = MU::Config::Ref.get(
566
567
  name: bastion['name'],
567
568
  cloud: vpc['cloud'],
@@ -596,7 +597,7 @@ module MU
596
597
  MU.log "Skipping malformed VPC peer in #{vpc['name']}", MU::ERR, details: peer
597
598
  next
598
599
  end
599
- peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
600
+ peer["#MU_CLOUDCLASS"] = MU::Cloud.loadBaseType("VPC")
600
601
  # We check for multiple siblings because some implementations
601
602
  # (Google) can split declared VPCs into parts to get the mimic the
602
603
  # routing behaviors we expect.
@@ -613,17 +614,11 @@ module MU
613
614
  append_me = { "vpc" => peer["vpc"].dup }
614
615
  append_me['vpc']['name'] = sib['name']
615
616
  append << append_me
616
- vpc["dependencies"] << {
617
- "type" => "vpc",
618
- "name" => sib['name']
619
- }
617
+ MU::Config.addDependency(vpc, sib['name'], "vpc", phase: "groom", no_create_wait: true)
620
618
  end
621
619
  delete << peer
622
620
  else
623
- vpc["dependencies"] << {
624
- "type" => "vpc",
625
- "name" => peer['vpc']["name"]
626
- }
621
+ MU::Config.addDependency(vpc, peer['vpc']['name'], "vpc", phase: "groom", no_create_wait: true)
627
622
  end
628
623
  delete << peer if sib['name'] == vpc['name']
629
624
  }
@@ -800,7 +795,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
800
795
  @@reference_cache[vpc_block] ||= ext_vpc if ok
801
796
  end
802
797
  rescue StandardError => e
803
- raise MuError, e.inspect, [caller, e.backtrace]
798
+ raise MuError.new e.inspect, details: { "my call stack" => caller, "exception call stack" => e.backtrace }
804
799
  ensure
805
800
  if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
806
801
  MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_type} #{parent['name']} (called by #{caller[0]})", MU::ERR, details: vpc_block
@@ -17,39 +17,39 @@ rhel71: &4
17
17
  us-west-1: ami-04898e596c06e802b
18
18
  us-west-2: ami-02db5457189a8a8c2
19
19
  centos6: &3
20
- us-east-1: ami-06b6c01abc6998348
21
- ap-northeast-1: ami-0c5da73fde2cb6437
22
- ap-northeast-2: ami-0134fce6dc00eb00d
23
- ap-south-1: ami-0e59a612e7c84836b
24
- ap-southeast-1: ami-0e16974f528ae0dae
25
- ap-southeast-2: ami-0e2feddf3dbf4d539
26
- ca-central-1: ami-089236a344dadad5f
27
- eu-central-1: ami-0c4eed3fe046c3917
28
- eu-north-1: ami-05f636e89d0362c14
29
- eu-west-1: ami-00c50b11d713f90d3
30
- eu-west-2: ami-06cc78c32eed7f944
31
- eu-west-3: ami-0ba626236ad786c54
32
- sa-east-1: ami-07c3b2a5a41e92376
33
- us-east-2: ami-01129e636778acfbc
34
- us-west-1: ami-0632e646cd5089ffc
35
- us-west-2: ami-0ce4c9f2e1037de53
20
+ us-east-1: ami-0ccdc671f12147a1d
21
+ us-east-2: ami-00d0e8bc2f05ab949
22
+ ap-northeast-1: ami-0726801ceef87f5f8
23
+ ap-northeast-2: ami-05fa4afc4a0493b0a
24
+ ap-south-1: ami-0d6e4f3b6592b3139
25
+ ap-southeast-1: ami-0c988e3dc80b14653
26
+ ap-southeast-2: ami-02ac856fd094675ef
27
+ ca-central-1: ami-0ce7e343953af2292
28
+ eu-central-1: ami-0ce8317423cea27b8
29
+ eu-north-1: ami-0a923b493d5fc9743
30
+ eu-west-1: ami-06e0f02328921c865
31
+ eu-west-2: ami-07ae118c8814df140
32
+ eu-west-3: ami-03c1017cd1ccc6e9d
33
+ sa-east-1: ami-05212ae133b9c3ba1
34
+ us-west-1: ami-0b05ec54412b9f8b0
35
+ us-west-2: ami-0447e036b102b2ca0
36
36
  centos7:
37
- us-east-1: ami-07e6f661e71ad964b
38
- ap-northeast-1: ami-0988180d74897c639
39
- ap-northeast-2: ami-0e77cd1c7024b8ae0
40
- ap-south-1: ami-02bd479122041000a
41
- ap-southeast-1: ami-017767778ef9db671
42
- ap-southeast-2: ami-05b09a58c3964d67d
43
- ca-central-1: ami-0a59a176d810fcc5f
44
- eu-central-1: ami-0b48a421fb05d96af
45
- eu-north-1: ami-02337601ea5dc4a5d
46
- eu-west-1: ami-0b0a55b7423eeac07
47
- eu-west-2: ami-060518b40b25b9eb4
48
- eu-west-3: ami-060957bb3adacd831
49
- sa-east-1: ami-0c706132b35071de6
50
- us-east-2: ami-0db4c266ed0bb958b
51
- us-west-1: ami-0980f6eb52c998793
52
- us-west-2: ami-07f2ed4755c01c05c
37
+ us-east-1: ami-067256ca1497c924d
38
+ ap-northeast-1: ami-07c1e51354fdfd362
39
+ ap-northeast-2: ami-042b761c93d6df2f1
40
+ ap-south-1: ami-02e879f52322e7c98
41
+ ap-southeast-1: ami-0487e9f84d0ffde89
42
+ ap-southeast-2: ami-0e854dab39fd6a427
43
+ ca-central-1: ami-05a27d311b585a70b
44
+ eu-central-1: ami-0e396d00c787b4f47
45
+ eu-north-1: ami-087763a2ba60b2bfe
46
+ eu-west-1: ami-04e3bd9335a14e635
47
+ eu-west-2: ami-0efd34a8d1fc2b104
48
+ eu-west-3: ami-08d0bcbc780448cf8
49
+ sa-east-1: ami-0284f4a0968263cf0
50
+ us-east-2: ami-0292786917d1e3015
51
+ us-west-1: ami-0ba622529dcdff2bb
52
+ us-west-2: ami-079a309ca6261d7f6
53
53
  ubuntu16: &2
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
@@ -2,6 +2,7 @@
2
2
  centos6: &centos6 OpenLogic/CentOS/6
3
3
  #centos7: &centos7 westernoceansoftwaresprivatelimited/centos-7-6/centos-7-6-server
4
4
  centos7: &centos7 OpenLogic/CentOS/7
5
+ centos8: &centos7 OpenLogic/CentOS/8
5
6
  rhel8: &rhel8 RedHat/RHEL/8
6
7
  rhel7: &rhel7 RedHat/RHEL/7
7
8
  rhel6: &rhel6 RedHat/RHEL/6
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  centos6: &centos6 egt-labs-admin/mu-centos-6
3
3
  centos7: &centos7 egt-labs-admin/mu-centos-7
4
+ centos8: &centos8 centos-cloud/centos-8
4
5
  rhel71: &rhel71 rhel-cloud/rhel-7
5
6
  rhel6: &rhel6 rhel-cloud/rhel-6
6
7
  debian10: &debian10 debian-cloud/debian-10
@@ -157,6 +157,7 @@ module MU
157
157
  _shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
158
158
  @main_config[data[:cfg_plural]].each { |resource|
159
159
  resource["#MU_CLOUDCLASS"] = classname
160
+ # resource["#MU_CLOUDCLASS"] = MU::Cloud.resourceClass(resource['cloud'], data[:cfg_plural])
160
161
  }
161
162
  setThreadDependencies(@main_config[data[:cfg_plural]])
162
163
  end
@@ -265,7 +266,7 @@ module MU
265
266
  # Run cloud provider-specific deploy meta-artifact creation (ssh keys,
266
267
  # resource groups, etc)
267
268
  @mommacat.cloudsUsed.each { |cloud|
268
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
269
+ cloudclass = MU::Cloud.cloudClass(cloud)
269
270
  cloudclass.initDeploy(@mommacat)
270
271
  }
271
272
 
@@ -554,16 +555,9 @@ MESSAGE_END
554
555
  @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
555
556
 
556
557
  MU.log "Setting dependencies for #{name}", MU::DEBUG, details: resource["dependencies"]
557
- if resource["dependencies"] != nil then
558
+ if !resource["dependencies"].nil? then
558
559
  resource["dependencies"].each { |dependency|
559
- parent_class = nil
560
- MU::Cloud.resource_types.each_pair { |res_class, attrs|
561
- if attrs[:cfg_name] == dependency['type'] or
562
- attrs[:cfg_plural] == dependency['type']
563
- parent_class = Object.const_get("MU").const_get("Cloud").const_get(res_class)
564
- break
565
- end
566
- }
560
+ parent_class = MU::Cloud.loadBaseType(dependency['type'])
567
561
 
568
562
  parent_type = parent_class.cfg_name
569
563
 
@@ -572,10 +566,10 @@ MESSAGE_END
572
566
  addDependentThread(parent, "#{name}_groom")
573
567
 
574
568
  # should our creation thread also wait on our parent's create?
575
- if !resource["no_create_wait"] and
569
+ if !dependency["no_create_wait"] and
576
570
  (resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
577
571
  dependency['phase'] == "create" or
578
- (parent_class.deps_wait_on_my_creation and parent_type != res_type))
572
+ parent_class.deps_wait_on_my_creation)
579
573
  addDependentThread(parent, "#{name}_create")
580
574
  end
581
575
 
@@ -584,12 +578,18 @@ MESSAGE_END
584
578
  if (dependency['phase'] == "groom" or resource["#MU_CLOUDCLASS"].waits_on_parent_completion) and parent_class.instance_methods(false).include?(:groom)
585
579
  parent = parent_type+"_"+dependency["name"]+"_groom"
586
580
  addDependentThread(parent, "#{name}_groom")
587
- if (parent_class.deps_wait_on_my_creation and parent_type != res_type) or resource["#MU_CLOUDCLASS"].waits_on_parent_completion or dependency['phase'] == "groom"
581
+ if !dependency["no_create_wait"] and (
582
+ parent_class.deps_wait_on_my_creation or
583
+ resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
584
+ dependency['phase'] == "groom"
585
+ )
588
586
  addDependentThread(parent, "#{name}_create")
589
587
  end
590
588
  end
591
589
  }
592
590
  end
591
+ MU.log "Thread dependencies #{res_type}[#{name}]", MU::DEBUG, details: { "create" => @dependency_threads["#{name}_create"], "groom" => @dependency_threads["#{name}_groom"] }
592
+ @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
593
593
  }
594
594
  end
595
595
 
@@ -651,8 +651,9 @@ MESSAGE_END
651
651
  run_this_method = myservice['#MUOBJECT'].method(mode)
652
652
  rescue StandardError => e
653
653
  MU::MommaCat.unlockAll
654
- @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
- raise e
654
+ @main_thread.raise MuError, "Error invoking #{myservice["#MUOBJECT"].class.name}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
+ return
656
+ # raise e
656
657
  end
657
658
  begin
658
659
  MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
@@ -30,6 +30,21 @@ module MU
30
30
  ["Chef", "Ansible"]
31
31
  end
32
32
 
33
+ # List of known/supported groomers which are installed and appear to be working
34
+ # @return [Array<String>]
35
+ def self.availableGroomers
36
+ available = []
37
+ MU::Groomer.supportedGroomers.each { |groomer|
38
+ begin
39
+ groomerbase = loadGroomer(groomer)
40
+ available << groomer if groomerbase.available?
41
+ rescue LoadError
42
+ end
43
+ }
44
+
45
+ available
46
+ end
47
+
33
48
  # Instance methods that any Groomer plugin must implement
34
49
  def self.requiredMethods
35
50
  [:preClean, :bootstrap, :haveBootstrapped?, :run, :saveDeployData, :getSecret, :saveSecret, :deleteSecret, :reinstall]
@@ -805,6 +805,9 @@ retry
805
805
  end
806
806
  end
807
807
 
808
+ # Purge Chef resources matching a particular deploy
809
+ # @param deploy_id [String]
810
+ # @param noop [Boolean]
808
811
  def self.cleanup(deploy_id, noop = false)
809
812
  return nil if deploy_id.nil? or deploy_id.empty?
810
813
  begin
@@ -33,6 +33,33 @@ module MU
33
33
  # Show DEBUG log entries and extra call stack and threading info
34
34
  LOUD = 2.freeze
35
35
 
36
+ # stash a hash map for color outputs
37
+ COLORMAP = {
38
+ MU::DEBUG => { :html => "orange", :ansi => :yellow },
39
+ MU::INFO => { :html => "green", :ansi => :green },
40
+ MU::NOTICE => { :html => "yellow", :ansi => :yellow },
41
+ MU::WARN => { :html => "orange", :ansi => :light_red },
42
+ MU::ERR => { :html => "red", :ansi => :red }
43
+ }.freeze
44
+
45
+ # minimum log verbosity at which we'll print various types of messages
46
+ PRINT_MSG_IF = {
47
+ MU::DEBUG => { :msg => LOUD, :details => LOUD },
48
+ MU::INFO => { :msg => NORMAL, :details => LOUD },
49
+ MU::NOTICE => { :msg => nil, :details => QUIET },
50
+ MU::WARN => { :msg => nil, :details => SILENT },
51
+ MU::ERR => { :msg => nil, :details => nil }
52
+ }.freeze
53
+
54
+ # Syslog equivalents of our log levels
55
+ SYSLOG_MAP = {
56
+ MU::DEBUG => Syslog::LOG_DEBUG,
57
+ MU::INFO => Syslog::LOG_NOTICE,
58
+ MU::NOTICE => Syslog::LOG_NOTICE,
59
+ MU::WARN => Syslog::LOG_WARNING,
60
+ MU::ERR => Syslog::LOG_ERR
61
+ }.freeze
62
+
36
63
  attr_accessor :verbosity
37
64
  @verbosity = MU::Logger::NORMAL
38
65
  @quiet = false
@@ -76,59 +103,28 @@ module MU
76
103
  html ||= @html
77
104
  handle ||= @handle
78
105
  color ||= @color
79
- return if verbosity == MU::Logger::SILENT
80
- return if verbosity < MU::Logger::LOUD and level == DEBUG
81
- return if verbosity < MU::Logger::NORMAL and level == INFO
82
106
 
83
- # By which we mean, "get the filename (with the .rb stripped off) which
84
- # originated the call to this method. Which, for our purposes, is the
85
- # MU subclass that called us. Useful information. And it looks like Perl.
86
- mod_root = Regexp.quote("#{ENV['MU_LIBDIR']}/modules/mu/")
87
- bin_root = Regexp.quote("#{ENV['MU_INSTALLDIR']}/bin/")
88
- caller_name = caller[1]
107
+ if verbosity == MU::Logger::SILENT or (verbosity < MU::Logger::LOUD and level == DEBUG) or (verbosity < MU::Logger::NORMAL and level == INFO)
108
+ return
109
+ end
89
110
 
90
- caller_name.sub!(/:.*/, "")
91
- caller_name.sub!(/^\.\//, "")
92
- caller_name.sub!(/^#{mod_root}/, "")
93
- caller_name.sub!(/^#{bin_root}/, "")
94
- caller_name.sub!(/\.r[ub]$/, "")
95
- caller_name.sub!(/#{Regexp.quote(MU.myRoot)}\//, "")
96
- caller_name.sub!(/^modules\//, "")
111
+ if level == SUMMARY
112
+ @summary << msg
113
+ return
114
+ end
115
+
116
+ caller_name = extract_caller_name(caller[1])
97
117
 
98
118
  time = Time.now.strftime("%b %d %H:%M:%S").to_s
99
119
 
100
120
  Syslog.open("Mu/"+caller_name, Syslog::LOG_PID, Syslog::LOG_DAEMON | Syslog::LOG_LOCAL3) if !Syslog.opened?
101
- if !details.nil?
102
- if details.is_a?(Hash) and details.has_key?(:details)
103
- details = details[:details]
104
- end
105
- details = PP.pp(details, '') if !details.is_a?(String)
106
- end
107
- details = "<pre>"+details+"</pre>" if html
108
- # We get passed literal quoted newlines sometimes, fix 'em. Get Windows'
109
- # ugly line feeds too.
110
- if !details.nil?
111
- details = details.dup # in case it's frozen or something
112
- details.gsub!(/\\n/, "\n")
113
- details.gsub!(/(\\r|\r)/, "")
114
- end
121
+
122
+ details = format_details(details, html)
115
123
 
116
124
  msg = msg.first if msg.is_a?(Array)
117
125
  msg = "" if msg == nil
118
126
  msg = msg.to_s if !msg.is_a?(String) and msg.respond_to?(:to_s)
119
127
 
120
- # wrapper for writing a log entry to multiple filehandles
121
- # @param handles [Array<IO>]
122
- # @param msgs [Array<String>]
123
- def write(handles = [], msgs = [])
124
- return if handles.nil? or msgs.nil?
125
- handles.each { |h|
126
- msgs.each { |m|
127
- h.puts m
128
- }
129
- }
130
- end
131
-
132
128
  @@log_semaphere.synchronize {
133
129
  handles = [handle]
134
130
  extra_logfile = if deploy and deploy.deploy_dir and Dir.exist?(deploy.deploy_dir)
@@ -137,110 +133,41 @@ module MU
137
133
  handles << extra_logfile if extra_logfile
138
134
  msgs = []
139
135
 
140
- case level
141
- when SUMMARY
142
- @summary << msg
143
- when DEBUG
144
- if verbosity >= MU::Logger::LOUD
145
- if html
146
- html_out "#{time} - #{caller_name} - #{msg}", "orange"
147
- html_out "&nbsp;#{details}" if details
148
- elsif color
149
- msgs << "#{time} - #{caller_name} - #{msg}".yellow.on_black
150
- msgs << "#{details}".white.on_black if details
151
- else
152
- msgs << "#{time} - #{caller_name} - #{msg}"
153
- msgs << "#{details}" if details
154
- end
155
- Syslog.log(Syslog::LOG_DEBUG, msg.gsub(/%/, ''))
156
- Syslog.log(Syslog::LOG_DEBUG, details.gsub(/%/, '')) if details
157
- end
158
- when INFO
159
- if verbosity >= MU::Logger::NORMAL
160
- if html
161
- html_out "#{time} - #{caller_name} - #{msg}", "green"
162
- elsif color
163
- msgs << "#{time} - #{caller_name} - #{msg}".green.on_black
164
- else
165
- msgs << "#{time} - #{caller_name} - #{msg}"
166
- end
167
- if verbosity >= MU::Logger::LOUD
168
- if html
169
- html_out "&nbsp;#{details}"
170
- elsif color
171
- msgs << "#{details}".white.on_black if details
172
- else
173
- msgs << "#{details}" if details
174
- end
175
- end
176
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
177
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
178
- end
179
- when NOTICE
180
- if html
181
- html_out "#{time} - #{caller_name} - #{msg}", "yellow"
182
- elsif color
183
- msgs << "#{time} - #{caller_name} - #{msg}".yellow.on_black
184
- else
185
- msgs << "#{time} - #{caller_name} - #{msg}"
186
- end
187
- if verbosity >= MU::Logger::QUIET
188
- if html
189
- html_out "#{caller_name} - #{msg}"
190
- elsif color
191
- msgs << "#{details}".white.on_black if details
192
- else
193
- msgs << "#{details}" if details
194
- end
195
- end
196
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
197
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
198
- when WARN
199
- if html
200
- html_out "#{time} - #{caller_name} - #{msg}", "orange"
201
- elsif color
202
- msgs << "#{time} - #{caller_name} - #{msg}".light_red.on_black
203
- else
204
- msgs << "#{time} - #{caller_name} - #{msg}"
205
- end
206
- if verbosity >= MU::Logger::SILENT
207
- if html
208
- html_out "#{caller_name} - #{msg}"
209
- elsif color
210
- msgs << "#{details}".white.on_black if details
211
- else
212
- msgs << "#{details}" if details
213
- end
214
- end
215
- Syslog.log(Syslog::LOG_WARNING, msg.gsub(/%/, ''))
216
- Syslog.log(Syslog::LOG_WARNING, details.gsub(/%/, '')) if details
217
- when ERR
218
- if html
219
- html_out "#{time} - #{caller_name} - #{msg}", "red"
220
- html_out "&nbsp;#{details}" if details
221
- elsif color
222
- msgs << "#{time} - #{caller_name} - #{msg}".red.on_black
223
- msgs << "#{details}".white.on_black if details
224
- else
225
- msgs << "#{time} - #{caller_name} - #{msg}"
226
- msgs << "#{details}" if details
227
- end
228
- Syslog.log(Syslog::LOG_ERR, msg.gsub(/%/, ''))
229
- Syslog.log(Syslog::LOG_ERR, details.gsub(/%/, '')) if details
136
+ if !PRINT_MSG_IF[level][:msg] or level >= PRINT_MSG_IF[level][:msg]
137
+ if html
138
+ html_out "#{time} - #{caller_name} - #{msg}", COLORMAP[level][:html]
139
+ else
140
+ str = "#{time} - #{caller_name} - #{msg}"
141
+ str = str.send(COLORMAP[level][:ansi]).on_black if color
142
+ msgs << str
143
+ end
144
+ Syslog.log(SYSLOG_MAP[level], msg.gsub(/%/, ''))
145
+ end
146
+
147
+ if details and (!PRINT_MSG_IF[level][:details] or level >= PRINT_MSG_IF[level][:details])
148
+ if html
149
+ html_out "&nbsp;#{details}"
230
150
  else
231
- if html
232
- html_out "#{time} - #{caller_name} - #{msg}"
233
- html_out "&nbsp;#{details}" if details
234
- elsif color
235
- msgs << "#{time} - #{caller_name} - #{msg}".white.on_black
236
- msgs << "#{details}".white.on_black if details
237
- else
238
- msgs << "#{time} - #{caller_name} - #{msg}"
239
- msgs << "#{details}" if details
240
- end
241
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
242
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
151
+ details = details.white.on_black if color
152
+ msgs << details
153
+ end
154
+ Syslog.log(SYSLOG_MAP[level], details.gsub(/%/, ''))
243
155
  end
156
+
157
+ # else
158
+ # if html
159
+ # html_out "#{time} - #{caller_name} - #{msg}"
160
+ # html_out "&nbsp;#{details}" if details
161
+ # elsif color
162
+ # msgs << "#{time} - #{caller_name} - #{msg}".white.on_black
163
+ # msgs << "#{details}".white.on_black if details
164
+ # else
165
+ # msgs << "#{time} - #{caller_name} - #{msg}"
166
+ # msgs << "#{details}" if details
167
+ # end
168
+ # Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
169
+ # Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
170
+
244
171
  write(handles, msgs)
245
172
 
246
173
  extra_logfile.close if extra_logfile
@@ -250,6 +177,43 @@ module MU
250
177
 
251
178
  private
252
179
 
180
+ def format_details(details, html = false)
181
+ return if details.nil?
182
+
183
+ if details.is_a?(Hash) and details.has_key?(:details)
184
+ details = details[:details]
185
+ end
186
+ details = PP.pp(details, '') if !details.is_a?(String)
187
+
188
+ details = "<pre>"+details+"</pre>" if html
189
+ # We get passed literal quoted newlines sometimes, fix 'em. Get Windows'
190
+ # ugly line feeds too.
191
+
192
+ details = details.dup # in case it's frozen or something
193
+ details.gsub!(/\\n/, "\n")
194
+ details.gsub!(/(\\r|\r)/, "")
195
+
196
+ details
197
+ end
198
+
199
+ # By which we mean, "get the filename (with the .rb stripped off) which
200
+ # originated the call to this method. Which, for our purposes, is the
201
+ # MU subclass that called us. Useful information. And it looks like Perl.
202
+ def extract_caller_name(caller_name)
203
+ return nil if !caller_name or !caller_name.is_a?(String)
204
+ mod_root = Regexp.quote("#{ENV['MU_LIBDIR']}/modules/mu/")
205
+ bin_root = Regexp.quote("#{ENV['MU_INSTALLDIR']}/bin/")
206
+
207
+ caller_name.sub!(/:.*/, "")
208
+ caller_name.sub!(/^\.\//, "")
209
+ caller_name.sub!(/^#{mod_root}/, "")
210
+ caller_name.sub!(/^#{bin_root}/, "")
211
+ caller_name.sub!(/\.r[ub]$/, "")
212
+ caller_name.sub!(/#{Regexp.quote(MU.myRoot)}\//, "")
213
+ caller_name.sub!(/^modules\//, "")
214
+ caller_name
215
+ end
216
+
253
217
  # Output a log message as HTML.
254
218
  #
255
219
  # @param msg [String]: The log message to print
@@ -259,5 +223,17 @@ module MU
259
223
  @handle.puts "<span style='color:#{rgb.css_rgb};'>#{msg}</span>"
260
224
  end
261
225
 
226
+ # wrapper for writing a log entry to multiple filehandles
227
+ # @param handles [Array<IO>]
228
+ # @param msgs [Array<String>]
229
+ def write(handles = [], msgs = [])
230
+ return if handles.nil? or msgs.nil?
231
+ handles.each { |h|
232
+ msgs.each { |m|
233
+ h.puts m
234
+ }
235
+ }
236
+ end
237
+
262
238
  end #class
263
239
  end #module