cloud-mu 3.1.6 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (154) hide show
  1. checksums.yaml +4 -4
  2. data/bin/mu-adopt +4 -12
  3. data/bin/mu-azure-tests +57 -0
  4. data/bin/mu-cleanup +2 -4
  5. data/bin/mu-configure +37 -1
  6. data/bin/mu-deploy +3 -3
  7. data/bin/mu-findstray-tests +25 -0
  8. data/bin/mu-gen-docs +2 -4
  9. data/bin/mu-run-tests +23 -10
  10. data/cloud-mu.gemspec +2 -2
  11. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  12. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  13. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  14. data/extras/generate-stock-images +1 -0
  15. data/modules/mu.rb +82 -95
  16. data/modules/mu/adoption.rb +356 -56
  17. data/modules/mu/cleanup.rb +21 -20
  18. data/modules/mu/cloud.rb +79 -1753
  19. data/modules/mu/cloud/database.rb +49 -0
  20. data/modules/mu/cloud/dnszone.rb +46 -0
  21. data/modules/mu/cloud/machine_images.rb +212 -0
  22. data/modules/mu/cloud/providers.rb +81 -0
  23. data/modules/mu/cloud/resource_base.rb +920 -0
  24. data/modules/mu/cloud/server.rb +40 -0
  25. data/modules/mu/cloud/server_pool.rb +1 -0
  26. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  27. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  28. data/modules/mu/cloud/wrappers.rb +165 -0
  29. data/modules/mu/config.rb +122 -80
  30. data/modules/mu/config/alarm.rb +2 -6
  31. data/modules/mu/config/bucket.rb +1 -1
  32. data/modules/mu/config/cache_cluster.rb +1 -1
  33. data/modules/mu/config/collection.rb +1 -1
  34. data/modules/mu/config/container_cluster.rb +2 -2
  35. data/modules/mu/config/database.rb +83 -104
  36. data/modules/mu/config/database.yml +1 -2
  37. data/modules/mu/config/dnszone.rb +1 -1
  38. data/modules/mu/config/doc_helpers.rb +4 -5
  39. data/modules/mu/config/endpoint.rb +1 -1
  40. data/modules/mu/config/firewall_rule.rb +3 -19
  41. data/modules/mu/config/folder.rb +1 -1
  42. data/modules/mu/config/function.rb +1 -1
  43. data/modules/mu/config/group.rb +1 -1
  44. data/modules/mu/config/habitat.rb +1 -1
  45. data/modules/mu/config/loadbalancer.rb +57 -11
  46. data/modules/mu/config/log.rb +1 -1
  47. data/modules/mu/config/msg_queue.rb +1 -1
  48. data/modules/mu/config/nosqldb.rb +1 -1
  49. data/modules/mu/config/notifier.rb +1 -1
  50. data/modules/mu/config/ref.rb +30 -4
  51. data/modules/mu/config/role.rb +1 -1
  52. data/modules/mu/config/schema_helpers.rb +30 -34
  53. data/modules/mu/config/search_domain.rb +1 -1
  54. data/modules/mu/config/server.rb +4 -12
  55. data/modules/mu/config/server_pool.rb +3 -7
  56. data/modules/mu/config/storage_pool.rb +1 -1
  57. data/modules/mu/config/tail.rb +10 -0
  58. data/modules/mu/config/user.rb +1 -1
  59. data/modules/mu/config/vpc.rb +12 -17
  60. data/modules/mu/defaults/AWS.yaml +32 -32
  61. data/modules/mu/defaults/Azure.yaml +1 -0
  62. data/modules/mu/defaults/Google.yaml +1 -0
  63. data/modules/mu/deploy.rb +16 -15
  64. data/modules/mu/groomer.rb +15 -0
  65. data/modules/mu/groomers/chef.rb +3 -0
  66. data/modules/mu/logger.rb +120 -144
  67. data/modules/mu/master.rb +1 -1
  68. data/modules/mu/mommacat.rb +54 -25
  69. data/modules/mu/mommacat/daemon.rb +10 -7
  70. data/modules/mu/mommacat/naming.rb +82 -3
  71. data/modules/mu/mommacat/search.rb +47 -15
  72. data/modules/mu/mommacat/storage.rb +72 -41
  73. data/modules/mu/{clouds → providers}/README.md +1 -1
  74. data/modules/mu/{clouds → providers}/aws.rb +114 -47
  75. data/modules/mu/{clouds → providers}/aws/alarm.rb +1 -1
  76. data/modules/mu/{clouds → providers}/aws/bucket.rb +2 -2
  77. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +10 -46
  78. data/modules/mu/{clouds → providers}/aws/collection.rb +3 -3
  79. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +15 -33
  80. data/modules/mu/providers/aws/database.rb +1744 -0
  81. data/modules/mu/{clouds → providers}/aws/dnszone.rb +2 -5
  82. data/modules/mu/{clouds → providers}/aws/endpoint.rb +2 -11
  83. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +33 -29
  84. data/modules/mu/{clouds → providers}/aws/folder.rb +0 -0
  85. data/modules/mu/{clouds → providers}/aws/function.rb +2 -10
  86. data/modules/mu/{clouds → providers}/aws/group.rb +9 -13
  87. data/modules/mu/{clouds → providers}/aws/habitat.rb +1 -1
  88. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +41 -33
  89. data/modules/mu/{clouds → providers}/aws/log.rb +2 -2
  90. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +2 -8
  91. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +0 -0
  92. data/modules/mu/{clouds → providers}/aws/notifier.rb +0 -0
  93. data/modules/mu/{clouds → providers}/aws/role.rb +7 -7
  94. data/modules/mu/{clouds → providers}/aws/search_domain.rb +8 -13
  95. data/modules/mu/{clouds → providers}/aws/server.rb +55 -90
  96. data/modules/mu/{clouds → providers}/aws/server_pool.rb +10 -33
  97. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +19 -36
  98. data/modules/mu/{clouds → providers}/aws/user.rb +8 -12
  99. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  100. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  101. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  102. data/modules/mu/{clouds → providers}/aws/vpc.rb +135 -70
  103. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  104. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  105. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  106. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  107. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  108. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  109. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  110. data/modules/mu/{clouds → providers}/azure/server.rb +30 -23
  111. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  112. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  113. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  114. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  115. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  116. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  117. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  118. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  119. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  120. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  121. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  122. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  123. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  124. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  125. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  126. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  127. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  128. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  129. data/modules/mu/{clouds → providers}/google.rb +14 -6
  130. data/modules/mu/{clouds → providers}/google/bucket.rb +1 -1
  131. data/modules/mu/{clouds → providers}/google/container_cluster.rb +28 -13
  132. data/modules/mu/{clouds → providers}/google/database.rb +1 -8
  133. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +2 -2
  134. data/modules/mu/{clouds → providers}/google/folder.rb +4 -8
  135. data/modules/mu/{clouds → providers}/google/function.rb +3 -3
  136. data/modules/mu/{clouds → providers}/google/group.rb +8 -16
  137. data/modules/mu/{clouds → providers}/google/habitat.rb +3 -7
  138. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +1 -1
  139. data/modules/mu/{clouds → providers}/google/role.rb +42 -34
  140. data/modules/mu/{clouds → providers}/google/server.rb +25 -10
  141. data/modules/mu/{clouds → providers}/google/server_pool.rb +10 -10
  142. data/modules/mu/{clouds → providers}/google/user.rb +31 -21
  143. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  144. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  145. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  146. data/modules/mu/{clouds → providers}/google/vpc.rb +37 -2
  147. data/modules/tests/centos6.yaml +11 -0
  148. data/modules/tests/centos7.yaml +11 -0
  149. data/modules/tests/centos8.yaml +12 -0
  150. data/modules/tests/rds.yaml +108 -0
  151. data/modules/tests/regrooms/rds.yaml +123 -0
  152. data/spec/mu/clouds/azure_spec.rb +2 -2
  153. metadata +108 -89
  154. data/modules/mu/clouds/aws/database.rb +0 -1974
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/storage_pool.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/storage_pool.rb
18
18
  class StoragePool
19
19
 
20
20
  # Base configuration schema for a StoragePool
@@ -117,6 +117,16 @@ module MU
117
117
  def gsub(*args)
118
118
  to_s.gsub(*args)
119
119
  end
120
+
121
+ # Lets callers access us like a {Hash}
122
+ # @param attribute [String,Symbol]
123
+ def [](attribute)
124
+ if respond_to?(attribute.to_sym)
125
+ send(attribute.to_sym)
126
+ else
127
+ nil
128
+ end
129
+ end
120
130
  end
121
131
 
122
132
  # Wrapper method for creating a {MU::Config::Tail} object as a reference to
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/user.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/user.rb
18
18
  class User
19
19
 
20
20
  # Base configuration schema for a User
@@ -14,7 +14,7 @@
14
14
 
15
15
  module MU
16
16
  class Config
17
- # Basket of Kittens config schema and parser logic. See modules/mu/clouds/*/vpc.rb
17
+ # Basket of Kittens config schema and parser logic. See modules/mu/providers/*/vpc.rb
18
18
  class VPC
19
19
 
20
20
  # Base configuration schema for a VPC
@@ -419,7 +419,7 @@ module MU
419
419
  if configurator.updating and configurator.existing_deploy and
420
420
  configurator.existing_deploy.original_config['vpcs']
421
421
  configurator.existing_deploy.original_config['vpcs'].each { |v|
422
- if v['name'] == vpc['name']
422
+ if v['name'].to_s == vpc['name'].to_s
423
423
  vpc['ip_block'] = v['ip_block']
424
424
  vpc['peers'] ||= []
425
425
  vpc['peers'].concat(v['peers'])
@@ -431,6 +431,10 @@ module MU
431
431
  break
432
432
  end
433
433
  }
434
+ if !vpc['ip_block']
435
+ MU.log "Loading existing deploy but can't find IP block of VPC #{vpc['name']}", MU::ERR
436
+ ok = false
437
+ end
434
438
  else
435
439
  using_default_cidr = true
436
440
  vpc['ip_block'] = "10.0.0.0/16"
@@ -539,7 +543,7 @@ module MU
539
543
  # Clouds that don't have some kind of native NAT gateway can also
540
544
  # leverage this host to honor "gateway" => "#NAT" situations.
541
545
  if !can_peer and !already_peered and have_public and vpc["create_bastion"]
542
- serverclass = Object.const_get("MU").const_get("Cloud").const_get(vpc["cloud"]).const_get("Server")
546
+ serverclass = MU::Cloud.resourceClass(vpc["cloud"], "Server")
543
547
  bastion = serverclass.genericNAT.dup
544
548
  bastion["groomer_variables"] = {
545
549
  "nat_ip_block" => vpc["ip_block"].to_s
@@ -558,10 +562,7 @@ module MU
558
562
  "name" => vpc["name"],
559
563
  "subnet_pref" => "public"
560
564
  }
561
- vpc["dependencies"] << {
562
- "type" => "server",
563
- "name" => bastion['name'],
564
- }
565
+ MU::Config.addDependency(vpc, bastion['name'], "server", no_create_wait: true)
565
566
  vpc["bastion"] = MU::Config::Ref.get(
566
567
  name: bastion['name'],
567
568
  cloud: vpc['cloud'],
@@ -596,7 +597,7 @@ module MU
596
597
  MU.log "Skipping malformed VPC peer in #{vpc['name']}", MU::ERR, details: peer
597
598
  next
598
599
  end
599
- peer["#MU_CLOUDCLASS"] = Object.const_get("MU").const_get("Cloud").const_get("VPC")
600
+ peer["#MU_CLOUDCLASS"] = MU::Cloud.loadBaseType("VPC")
600
601
  # We check for multiple siblings because some implementations
601
602
  # (Google) can split declared VPCs into parts to get the mimic the
602
603
  # routing behaviors we expect.
@@ -613,17 +614,11 @@ module MU
613
614
  append_me = { "vpc" => peer["vpc"].dup }
614
615
  append_me['vpc']['name'] = sib['name']
615
616
  append << append_me
616
- vpc["dependencies"] << {
617
- "type" => "vpc",
618
- "name" => sib['name']
619
- }
617
+ MU::Config.addDependency(vpc, sib['name'], "vpc", phase: "groom", no_create_wait: true)
620
618
  end
621
619
  delete << peer
622
620
  else
623
- vpc["dependencies"] << {
624
- "type" => "vpc",
625
- "name" => peer['vpc']["name"]
626
- }
621
+ MU::Config.addDependency(vpc, peer['vpc']['name'], "vpc", phase: "groom", no_create_wait: true)
627
622
  end
628
623
  delete << peer if sib['name'] == vpc['name']
629
624
  }
@@ -800,7 +795,7 @@ MU.log "VPC lookup cache hit", MU::WARN, details: vpc_block
800
795
  @@reference_cache[vpc_block] ||= ext_vpc if ok
801
796
  end
802
797
  rescue StandardError => e
803
- raise MuError, e.inspect, [caller, e.backtrace]
798
+ raise MuError.new e.inspect, details: { "my call stack" => caller, "exception call stack" => e.backtrace }
804
799
  ensure
805
800
  if !ext_vpc and vpc_block['cloud'] != "CloudFormation"
806
801
  MU.log "Couldn't resolve VPC reference to a unique live VPC in #{parent_type} #{parent['name']} (called by #{caller[0]})", MU::ERR, details: vpc_block
@@ -17,39 +17,39 @@ rhel71: &4
17
17
  us-west-1: ami-04898e596c06e802b
18
18
  us-west-2: ami-02db5457189a8a8c2
19
19
  centos6: &3
20
- us-east-1: ami-06b6c01abc6998348
21
- ap-northeast-1: ami-0c5da73fde2cb6437
22
- ap-northeast-2: ami-0134fce6dc00eb00d
23
- ap-south-1: ami-0e59a612e7c84836b
24
- ap-southeast-1: ami-0e16974f528ae0dae
25
- ap-southeast-2: ami-0e2feddf3dbf4d539
26
- ca-central-1: ami-089236a344dadad5f
27
- eu-central-1: ami-0c4eed3fe046c3917
28
- eu-north-1: ami-05f636e89d0362c14
29
- eu-west-1: ami-00c50b11d713f90d3
30
- eu-west-2: ami-06cc78c32eed7f944
31
- eu-west-3: ami-0ba626236ad786c54
32
- sa-east-1: ami-07c3b2a5a41e92376
33
- us-east-2: ami-01129e636778acfbc
34
- us-west-1: ami-0632e646cd5089ffc
35
- us-west-2: ami-0ce4c9f2e1037de53
20
+ us-east-1: ami-0ccdc671f12147a1d
21
+ us-east-2: ami-00d0e8bc2f05ab949
22
+ ap-northeast-1: ami-0726801ceef87f5f8
23
+ ap-northeast-2: ami-05fa4afc4a0493b0a
24
+ ap-south-1: ami-0d6e4f3b6592b3139
25
+ ap-southeast-1: ami-0c988e3dc80b14653
26
+ ap-southeast-2: ami-02ac856fd094675ef
27
+ ca-central-1: ami-0ce7e343953af2292
28
+ eu-central-1: ami-0ce8317423cea27b8
29
+ eu-north-1: ami-0a923b493d5fc9743
30
+ eu-west-1: ami-06e0f02328921c865
31
+ eu-west-2: ami-07ae118c8814df140
32
+ eu-west-3: ami-03c1017cd1ccc6e9d
33
+ sa-east-1: ami-05212ae133b9c3ba1
34
+ us-west-1: ami-0b05ec54412b9f8b0
35
+ us-west-2: ami-0447e036b102b2ca0
36
36
  centos7:
37
- us-east-1: ami-07e6f661e71ad964b
38
- ap-northeast-1: ami-0988180d74897c639
39
- ap-northeast-2: ami-0e77cd1c7024b8ae0
40
- ap-south-1: ami-02bd479122041000a
41
- ap-southeast-1: ami-017767778ef9db671
42
- ap-southeast-2: ami-05b09a58c3964d67d
43
- ca-central-1: ami-0a59a176d810fcc5f
44
- eu-central-1: ami-0b48a421fb05d96af
45
- eu-north-1: ami-02337601ea5dc4a5d
46
- eu-west-1: ami-0b0a55b7423eeac07
47
- eu-west-2: ami-060518b40b25b9eb4
48
- eu-west-3: ami-060957bb3adacd831
49
- sa-east-1: ami-0c706132b35071de6
50
- us-east-2: ami-0db4c266ed0bb958b
51
- us-west-1: ami-0980f6eb52c998793
52
- us-west-2: ami-07f2ed4755c01c05c
37
+ us-east-1: ami-067256ca1497c924d
38
+ ap-northeast-1: ami-07c1e51354fdfd362
39
+ ap-northeast-2: ami-042b761c93d6df2f1
40
+ ap-south-1: ami-02e879f52322e7c98
41
+ ap-southeast-1: ami-0487e9f84d0ffde89
42
+ ap-southeast-2: ami-0e854dab39fd6a427
43
+ ca-central-1: ami-05a27d311b585a70b
44
+ eu-central-1: ami-0e396d00c787b4f47
45
+ eu-north-1: ami-087763a2ba60b2bfe
46
+ eu-west-1: ami-04e3bd9335a14e635
47
+ eu-west-2: ami-0efd34a8d1fc2b104
48
+ eu-west-3: ami-08d0bcbc780448cf8
49
+ sa-east-1: ami-0284f4a0968263cf0
50
+ us-east-2: ami-0292786917d1e3015
51
+ us-west-1: ami-0ba622529dcdff2bb
52
+ us-west-2: ami-079a309ca6261d7f6
53
53
  ubuntu16: &2
54
54
  us-east-1: ami-bcdc16c6
55
55
  us-west-1: ami-1b17257b
@@ -2,6 +2,7 @@
2
2
  centos6: &centos6 OpenLogic/CentOS/6
3
3
  #centos7: &centos7 westernoceansoftwaresprivatelimited/centos-7-6/centos-7-6-server
4
4
  centos7: &centos7 OpenLogic/CentOS/7
5
+ centos8: &centos7 OpenLogic/CentOS/8
5
6
  rhel8: &rhel8 RedHat/RHEL/8
6
7
  rhel7: &rhel7 RedHat/RHEL/7
7
8
  rhel6: &rhel6 RedHat/RHEL/6
@@ -1,6 +1,7 @@
1
1
  ---
2
2
  centos6: &centos6 egt-labs-admin/mu-centos-6
3
3
  centos7: &centos7 egt-labs-admin/mu-centos-7
4
+ centos8: &centos8 centos-cloud/centos-8
4
5
  rhel71: &rhel71 rhel-cloud/rhel-7
5
6
  rhel6: &rhel6 rhel-cloud/rhel-6
6
7
  debian10: &debian10 debian-cloud/debian-10
@@ -157,6 +157,7 @@ module MU
157
157
  _shortclass, _cfg_name, _cfg_plural, classname = MU::Cloud.getResourceNames(data[:cfg_plural])
158
158
  @main_config[data[:cfg_plural]].each { |resource|
159
159
  resource["#MU_CLOUDCLASS"] = classname
160
+ # resource["#MU_CLOUDCLASS"] = MU::Cloud.resourceClass(resource['cloud'], data[:cfg_plural])
160
161
  }
161
162
  setThreadDependencies(@main_config[data[:cfg_plural]])
162
163
  end
@@ -265,7 +266,7 @@ module MU
265
266
  # Run cloud provider-specific deploy meta-artifact creation (ssh keys,
266
267
  # resource groups, etc)
267
268
  @mommacat.cloudsUsed.each { |cloud|
268
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
269
+ cloudclass = MU::Cloud.cloudClass(cloud)
269
270
  cloudclass.initDeploy(@mommacat)
270
271
  }
271
272
 
@@ -554,16 +555,9 @@ MESSAGE_END
554
555
  @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
555
556
 
556
557
  MU.log "Setting dependencies for #{name}", MU::DEBUG, details: resource["dependencies"]
557
- if resource["dependencies"] != nil then
558
+ if !resource["dependencies"].nil? then
558
559
  resource["dependencies"].each { |dependency|
559
- parent_class = nil
560
- MU::Cloud.resource_types.each_pair { |res_class, attrs|
561
- if attrs[:cfg_name] == dependency['type'] or
562
- attrs[:cfg_plural] == dependency['type']
563
- parent_class = Object.const_get("MU").const_get("Cloud").const_get(res_class)
564
- break
565
- end
566
- }
560
+ parent_class = MU::Cloud.loadBaseType(dependency['type'])
567
561
 
568
562
  parent_type = parent_class.cfg_name
569
563
 
@@ -572,10 +566,10 @@ MESSAGE_END
572
566
  addDependentThread(parent, "#{name}_groom")
573
567
 
574
568
  # should our creation thread also wait on our parent's create?
575
- if !resource["no_create_wait"] and
569
+ if !dependency["no_create_wait"] and
576
570
  (resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
577
571
  dependency['phase'] == "create" or
578
- (parent_class.deps_wait_on_my_creation and parent_type != res_type))
572
+ parent_class.deps_wait_on_my_creation)
579
573
  addDependentThread(parent, "#{name}_create")
580
574
  end
581
575
 
@@ -584,12 +578,18 @@ MESSAGE_END
584
578
  if (dependency['phase'] == "groom" or resource["#MU_CLOUDCLASS"].waits_on_parent_completion) and parent_class.instance_methods(false).include?(:groom)
585
579
  parent = parent_type+"_"+dependency["name"]+"_groom"
586
580
  addDependentThread(parent, "#{name}_groom")
587
- if (parent_class.deps_wait_on_my_creation and parent_type != res_type) or resource["#MU_CLOUDCLASS"].waits_on_parent_completion or dependency['phase'] == "groom"
581
+ if !dependency["no_create_wait"] and (
582
+ parent_class.deps_wait_on_my_creation or
583
+ resource["#MU_CLOUDCLASS"].waits_on_parent_completion or
584
+ dependency['phase'] == "groom"
585
+ )
588
586
  addDependentThread(parent, "#{name}_create")
589
587
  end
590
588
  end
591
589
  }
592
590
  end
591
+ MU.log "Thread dependencies #{res_type}[#{name}]", MU::DEBUG, details: { "create" => @dependency_threads["#{name}_create"], "groom" => @dependency_threads["#{name}_groom"] }
592
+ @dependency_threads["#{name}_groom"]=["#{name}_create", "mu_groom_container"]
593
593
  }
594
594
  end
595
595
 
@@ -651,8 +651,9 @@ MESSAGE_END
651
651
  run_this_method = myservice['#MUOBJECT'].method(mode)
652
652
  rescue StandardError => e
653
653
  MU::MommaCat.unlockAll
654
- @main_thread.raise MuError, "Error invoking #{myservice["#MU_CLOUDCLASS"]}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
- raise e
654
+ @main_thread.raise MuError, "Error invoking #{myservice["#MUOBJECT"].class.name}.#{mode} for #{myservice['name']} (#{e.inspect})", e.backtrace
655
+ return
656
+ # raise e
656
657
  end
657
658
  begin
658
659
  MU.log "Checking whether to run #{myservice['#MUOBJECT']}.#{mode} (updating: #{@updating})", MU::DEBUG
@@ -30,6 +30,21 @@ module MU
30
30
  ["Chef", "Ansible"]
31
31
  end
32
32
 
33
+ # List of known/supported groomers which are installed and appear to be working
34
+ # @return [Array<String>]
35
+ def self.availableGroomers
36
+ available = []
37
+ MU::Groomer.supportedGroomers.each { |groomer|
38
+ begin
39
+ groomerbase = loadGroomer(groomer)
40
+ available << groomer if groomerbase.available?
41
+ rescue LoadError
42
+ end
43
+ }
44
+
45
+ available
46
+ end
47
+
33
48
  # Instance methods that any Groomer plugin must implement
34
49
  def self.requiredMethods
35
50
  [:preClean, :bootstrap, :haveBootstrapped?, :run, :saveDeployData, :getSecret, :saveSecret, :deleteSecret, :reinstall]
@@ -805,6 +805,9 @@ retry
805
805
  end
806
806
  end
807
807
 
808
+ # Purge Chef resources matching a particular deploy
809
+ # @param deploy_id [String]
810
+ # @param noop [Boolean]
808
811
  def self.cleanup(deploy_id, noop = false)
809
812
  return nil if deploy_id.nil? or deploy_id.empty?
810
813
  begin
@@ -33,6 +33,33 @@ module MU
33
33
  # Show DEBUG log entries and extra call stack and threading info
34
34
  LOUD = 2.freeze
35
35
 
36
+ # stash a hash map for color outputs
37
+ COLORMAP = {
38
+ MU::DEBUG => { :html => "orange", :ansi => :yellow },
39
+ MU::INFO => { :html => "green", :ansi => :green },
40
+ MU::NOTICE => { :html => "yellow", :ansi => :yellow },
41
+ MU::WARN => { :html => "orange", :ansi => :light_red },
42
+ MU::ERR => { :html => "red", :ansi => :red }
43
+ }.freeze
44
+
45
+ # minimum log verbosity at which we'll print various types of messages
46
+ PRINT_MSG_IF = {
47
+ MU::DEBUG => { :msg => LOUD, :details => LOUD },
48
+ MU::INFO => { :msg => NORMAL, :details => LOUD },
49
+ MU::NOTICE => { :msg => nil, :details => QUIET },
50
+ MU::WARN => { :msg => nil, :details => SILENT },
51
+ MU::ERR => { :msg => nil, :details => nil }
52
+ }.freeze
53
+
54
+ # Syslog equivalents of our log levels
55
+ SYSLOG_MAP = {
56
+ MU::DEBUG => Syslog::LOG_DEBUG,
57
+ MU::INFO => Syslog::LOG_NOTICE,
58
+ MU::NOTICE => Syslog::LOG_NOTICE,
59
+ MU::WARN => Syslog::LOG_WARNING,
60
+ MU::ERR => Syslog::LOG_ERR
61
+ }.freeze
62
+
36
63
  attr_accessor :verbosity
37
64
  @verbosity = MU::Logger::NORMAL
38
65
  @quiet = false
@@ -76,59 +103,28 @@ module MU
76
103
  html ||= @html
77
104
  handle ||= @handle
78
105
  color ||= @color
79
- return if verbosity == MU::Logger::SILENT
80
- return if verbosity < MU::Logger::LOUD and level == DEBUG
81
- return if verbosity < MU::Logger::NORMAL and level == INFO
82
106
 
83
- # By which we mean, "get the filename (with the .rb stripped off) which
84
- # originated the call to this method. Which, for our purposes, is the
85
- # MU subclass that called us. Useful information. And it looks like Perl.
86
- mod_root = Regexp.quote("#{ENV['MU_LIBDIR']}/modules/mu/")
87
- bin_root = Regexp.quote("#{ENV['MU_INSTALLDIR']}/bin/")
88
- caller_name = caller[1]
107
+ if verbosity == MU::Logger::SILENT or (verbosity < MU::Logger::LOUD and level == DEBUG) or (verbosity < MU::Logger::NORMAL and level == INFO)
108
+ return
109
+ end
89
110
 
90
- caller_name.sub!(/:.*/, "")
91
- caller_name.sub!(/^\.\//, "")
92
- caller_name.sub!(/^#{mod_root}/, "")
93
- caller_name.sub!(/^#{bin_root}/, "")
94
- caller_name.sub!(/\.r[ub]$/, "")
95
- caller_name.sub!(/#{Regexp.quote(MU.myRoot)}\//, "")
96
- caller_name.sub!(/^modules\//, "")
111
+ if level == SUMMARY
112
+ @summary << msg
113
+ return
114
+ end
115
+
116
+ caller_name = extract_caller_name(caller[1])
97
117
 
98
118
  time = Time.now.strftime("%b %d %H:%M:%S").to_s
99
119
 
100
120
  Syslog.open("Mu/"+caller_name, Syslog::LOG_PID, Syslog::LOG_DAEMON | Syslog::LOG_LOCAL3) if !Syslog.opened?
101
- if !details.nil?
102
- if details.is_a?(Hash) and details.has_key?(:details)
103
- details = details[:details]
104
- end
105
- details = PP.pp(details, '') if !details.is_a?(String)
106
- end
107
- details = "<pre>"+details+"</pre>" if html
108
- # We get passed literal quoted newlines sometimes, fix 'em. Get Windows'
109
- # ugly line feeds too.
110
- if !details.nil?
111
- details = details.dup # in case it's frozen or something
112
- details.gsub!(/\\n/, "\n")
113
- details.gsub!(/(\\r|\r)/, "")
114
- end
121
+
122
+ details = format_details(details, html)
115
123
 
116
124
  msg = msg.first if msg.is_a?(Array)
117
125
  msg = "" if msg == nil
118
126
  msg = msg.to_s if !msg.is_a?(String) and msg.respond_to?(:to_s)
119
127
 
120
- # wrapper for writing a log entry to multiple filehandles
121
- # @param handles [Array<IO>]
122
- # @param msgs [Array<String>]
123
- def write(handles = [], msgs = [])
124
- return if handles.nil? or msgs.nil?
125
- handles.each { |h|
126
- msgs.each { |m|
127
- h.puts m
128
- }
129
- }
130
- end
131
-
132
128
  @@log_semaphere.synchronize {
133
129
  handles = [handle]
134
130
  extra_logfile = if deploy and deploy.deploy_dir and Dir.exist?(deploy.deploy_dir)
@@ -137,110 +133,41 @@ module MU
137
133
  handles << extra_logfile if extra_logfile
138
134
  msgs = []
139
135
 
140
- case level
141
- when SUMMARY
142
- @summary << msg
143
- when DEBUG
144
- if verbosity >= MU::Logger::LOUD
145
- if html
146
- html_out "#{time} - #{caller_name} - #{msg}", "orange"
147
- html_out "&nbsp;#{details}" if details
148
- elsif color
149
- msgs << "#{time} - #{caller_name} - #{msg}".yellow.on_black
150
- msgs << "#{details}".white.on_black if details
151
- else
152
- msgs << "#{time} - #{caller_name} - #{msg}"
153
- msgs << "#{details}" if details
154
- end
155
- Syslog.log(Syslog::LOG_DEBUG, msg.gsub(/%/, ''))
156
- Syslog.log(Syslog::LOG_DEBUG, details.gsub(/%/, '')) if details
157
- end
158
- when INFO
159
- if verbosity >= MU::Logger::NORMAL
160
- if html
161
- html_out "#{time} - #{caller_name} - #{msg}", "green"
162
- elsif color
163
- msgs << "#{time} - #{caller_name} - #{msg}".green.on_black
164
- else
165
- msgs << "#{time} - #{caller_name} - #{msg}"
166
- end
167
- if verbosity >= MU::Logger::LOUD
168
- if html
169
- html_out "&nbsp;#{details}"
170
- elsif color
171
- msgs << "#{details}".white.on_black if details
172
- else
173
- msgs << "#{details}" if details
174
- end
175
- end
176
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
177
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
178
- end
179
- when NOTICE
180
- if html
181
- html_out "#{time} - #{caller_name} - #{msg}", "yellow"
182
- elsif color
183
- msgs << "#{time} - #{caller_name} - #{msg}".yellow.on_black
184
- else
185
- msgs << "#{time} - #{caller_name} - #{msg}"
186
- end
187
- if verbosity >= MU::Logger::QUIET
188
- if html
189
- html_out "#{caller_name} - #{msg}"
190
- elsif color
191
- msgs << "#{details}".white.on_black if details
192
- else
193
- msgs << "#{details}" if details
194
- end
195
- end
196
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
197
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
198
- when WARN
199
- if html
200
- html_out "#{time} - #{caller_name} - #{msg}", "orange"
201
- elsif color
202
- msgs << "#{time} - #{caller_name} - #{msg}".light_red.on_black
203
- else
204
- msgs << "#{time} - #{caller_name} - #{msg}"
205
- end
206
- if verbosity >= MU::Logger::SILENT
207
- if html
208
- html_out "#{caller_name} - #{msg}"
209
- elsif color
210
- msgs << "#{details}".white.on_black if details
211
- else
212
- msgs << "#{details}" if details
213
- end
214
- end
215
- Syslog.log(Syslog::LOG_WARNING, msg.gsub(/%/, ''))
216
- Syslog.log(Syslog::LOG_WARNING, details.gsub(/%/, '')) if details
217
- when ERR
218
- if html
219
- html_out "#{time} - #{caller_name} - #{msg}", "red"
220
- html_out "&nbsp;#{details}" if details
221
- elsif color
222
- msgs << "#{time} - #{caller_name} - #{msg}".red.on_black
223
- msgs << "#{details}".white.on_black if details
224
- else
225
- msgs << "#{time} - #{caller_name} - #{msg}"
226
- msgs << "#{details}" if details
227
- end
228
- Syslog.log(Syslog::LOG_ERR, msg.gsub(/%/, ''))
229
- Syslog.log(Syslog::LOG_ERR, details.gsub(/%/, '')) if details
136
+ if !PRINT_MSG_IF[level][:msg] or level >= PRINT_MSG_IF[level][:msg]
137
+ if html
138
+ html_out "#{time} - #{caller_name} - #{msg}", COLORMAP[level][:html]
139
+ else
140
+ str = "#{time} - #{caller_name} - #{msg}"
141
+ str = str.send(COLORMAP[level][:ansi]).on_black if color
142
+ msgs << str
143
+ end
144
+ Syslog.log(SYSLOG_MAP[level], msg.gsub(/%/, ''))
145
+ end
146
+
147
+ if details and (!PRINT_MSG_IF[level][:details] or level >= PRINT_MSG_IF[level][:details])
148
+ if html
149
+ html_out "&nbsp;#{details}"
230
150
  else
231
- if html
232
- html_out "#{time} - #{caller_name} - #{msg}"
233
- html_out "&nbsp;#{details}" if details
234
- elsif color
235
- msgs << "#{time} - #{caller_name} - #{msg}".white.on_black
236
- msgs << "#{details}".white.on_black if details
237
- else
238
- msgs << "#{time} - #{caller_name} - #{msg}"
239
- msgs << "#{details}" if details
240
- end
241
- Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
242
- Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
151
+ details = details.white.on_black if color
152
+ msgs << details
153
+ end
154
+ Syslog.log(SYSLOG_MAP[level], details.gsub(/%/, ''))
243
155
  end
156
+
157
+ # else
158
+ # if html
159
+ # html_out "#{time} - #{caller_name} - #{msg}"
160
+ # html_out "&nbsp;#{details}" if details
161
+ # elsif color
162
+ # msgs << "#{time} - #{caller_name} - #{msg}".white.on_black
163
+ # msgs << "#{details}".white.on_black if details
164
+ # else
165
+ # msgs << "#{time} - #{caller_name} - #{msg}"
166
+ # msgs << "#{details}" if details
167
+ # end
168
+ # Syslog.log(Syslog::LOG_NOTICE, msg.gsub(/%/, ''))
169
+ # Syslog.log(Syslog::LOG_NOTICE, details.gsub(/%/, '')) if details
170
+
244
171
  write(handles, msgs)
245
172
 
246
173
  extra_logfile.close if extra_logfile
@@ -250,6 +177,43 @@ module MU
250
177
 
251
178
  private
252
179
 
180
+ def format_details(details, html = false)
181
+ return if details.nil?
182
+
183
+ if details.is_a?(Hash) and details.has_key?(:details)
184
+ details = details[:details]
185
+ end
186
+ details = PP.pp(details, '') if !details.is_a?(String)
187
+
188
+ details = "<pre>"+details+"</pre>" if html
189
+ # We get passed literal quoted newlines sometimes, fix 'em. Get Windows'
190
+ # ugly line feeds too.
191
+
192
+ details = details.dup # in case it's frozen or something
193
+ details.gsub!(/\\n/, "\n")
194
+ details.gsub!(/(\\r|\r)/, "")
195
+
196
+ details
197
+ end
198
+
199
+ # By which we mean, "get the filename (with the .rb stripped off) which
200
+ # originated the call to this method. Which, for our purposes, is the
201
+ # MU subclass that called us. Useful information. And it looks like Perl.
202
+ def extract_caller_name(caller_name)
203
+ return nil if !caller_name or !caller_name.is_a?(String)
204
+ mod_root = Regexp.quote("#{ENV['MU_LIBDIR']}/modules/mu/")
205
+ bin_root = Regexp.quote("#{ENV['MU_INSTALLDIR']}/bin/")
206
+
207
+ caller_name.sub!(/:.*/, "")
208
+ caller_name.sub!(/^\.\//, "")
209
+ caller_name.sub!(/^#{mod_root}/, "")
210
+ caller_name.sub!(/^#{bin_root}/, "")
211
+ caller_name.sub!(/\.r[ub]$/, "")
212
+ caller_name.sub!(/#{Regexp.quote(MU.myRoot)}\//, "")
213
+ caller_name.sub!(/^modules\//, "")
214
+ caller_name
215
+ end
216
+
253
217
  # Output a log message as HTML.
254
218
  #
255
219
  # @param msg [String]: The log message to print
@@ -259,5 +223,17 @@ module MU
259
223
  @handle.puts "<span style='color:#{rgb.css_rgb};'>#{msg}</span>"
260
224
  end
261
225
 
226
+ # wrapper for writing a log entry to multiple filehandles
227
+ # @param handles [Array<IO>]
228
+ # @param msgs [Array<String>]
229
+ def write(handles = [], msgs = [])
230
+ return if handles.nil? or msgs.nil?
231
+ handles.each { |h|
232
+ msgs.each { |m|
233
+ h.puts m
234
+ }
235
+ }
236
+ end
237
+
262
238
  end #class
263
239
  end #module