cloud-mu 3.5.0 → 3.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +5 -2
  3. data/Berksfile.lock +135 -0
  4. data/ansible/roles/mu-base/README.md +33 -0
  5. data/ansible/roles/mu-base/defaults/main.yml +2 -0
  6. data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
  7. data/ansible/roles/mu-base/files/check_apm.sh +18 -0
  8. data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
  9. data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
  10. data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
  11. data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
  12. data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
  13. data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
  14. data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
  15. data/ansible/roles/mu-base/files/logrotate.conf +35 -0
  16. data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
  17. data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
  18. data/ansible/roles/mu-base/handlers/main.yml +5 -0
  19. data/ansible/roles/mu-base/meta/main.yml +53 -0
  20. data/ansible/roles/mu-base/tasks/main.yml +113 -0
  21. data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
  22. data/ansible/roles/mu-base/tests/inventory +2 -0
  23. data/ansible/roles/mu-base/tests/test.yml +5 -0
  24. data/ansible/roles/mu-base/vars/main.yml +1 -0
  25. data/ansible/roles/mu-compliance/README.md +33 -0
  26. data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
  27. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
  28. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
  29. data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
  30. data/ansible/roles/mu-compliance/meta/main.yml +53 -0
  31. data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
  32. data/ansible/roles/mu-compliance/tests/inventory +2 -0
  33. data/ansible/roles/mu-compliance/tests/test.yml +5 -0
  34. data/ansible/roles/mu-compliance/vars/main.yml +4 -0
  35. data/ansible/roles/mu-elastic/README.md +51 -0
  36. data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
  37. data/ansible/roles/mu-elastic/files/jvm.options +93 -0
  38. data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
  39. data/ansible/roles/mu-elastic/meta/main.yml +52 -0
  40. data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
  41. data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
  42. data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
  43. data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
  44. data/ansible/roles/mu-elastic/tests/inventory +2 -0
  45. data/ansible/roles/mu-elastic/tests/test.yml +5 -0
  46. data/ansible/roles/mu-elastic/vars/main.yml +2 -0
  47. data/ansible/roles/mu-logstash/README.md +51 -0
  48. data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
  49. data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
  50. data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
  51. data/ansible/roles/mu-logstash/files/jvm.options +84 -0
  52. data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
  53. data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
  54. data/ansible/roles/mu-logstash/meta/main.yml +52 -0
  55. data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
  56. data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
  57. data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
  58. data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
  59. data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
  60. data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
  61. data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
  62. data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
  63. data/ansible/roles/mu-logstash/tests/inventory +2 -0
  64. data/ansible/roles/mu-logstash/tests/test.yml +5 -0
  65. data/ansible/roles/mu-logstash/vars/main.yml +2 -0
  66. data/ansible/roles/mu-rdp/README.md +33 -0
  67. data/ansible/roles/mu-rdp/meta/main.yml +53 -0
  68. data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
  69. data/ansible/roles/mu-rdp/tests/inventory +2 -0
  70. data/ansible/roles/mu-rdp/tests/test.yml +5 -0
  71. data/ansible/roles/mu-windows/tasks/main.yml +3 -0
  72. data/bin/mu-ansible-secret +1 -1
  73. data/bin/mu-aws-setup +4 -3
  74. data/bin/mu-azure-setup +5 -5
  75. data/bin/mu-configure +25 -17
  76. data/bin/mu-firewall-allow-clients +1 -0
  77. data/bin/mu-gcp-setup +3 -3
  78. data/bin/mu-load-config.rb +1 -0
  79. data/bin/mu-node-manage +66 -33
  80. data/bin/mu-self-update +2 -2
  81. data/bin/mu-upload-chef-artifacts +6 -1
  82. data/bin/mu-user-manage +1 -1
  83. data/cloud-mu.gemspec +25 -23
  84. data/cookbooks/firewall/CHANGELOG.md +417 -224
  85. data/cookbooks/firewall/LICENSE +202 -0
  86. data/cookbooks/firewall/README.md +153 -126
  87. data/cookbooks/firewall/TODO.md +6 -0
  88. data/cookbooks/firewall/attributes/firewalld.rb +7 -0
  89. data/cookbooks/firewall/attributes/iptables.rb +3 -3
  90. data/cookbooks/firewall/chefignore +115 -0
  91. data/cookbooks/firewall/libraries/helpers.rb +5 -0
  92. data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
  93. data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
  94. data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
  95. data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
  96. data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
  97. data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
  98. data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
  99. data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
  100. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
  101. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
  102. data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
  103. data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
  104. data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
  105. data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
  106. data/cookbooks/firewall/metadata.json +40 -1
  107. data/cookbooks/firewall/metadata.rb +15 -0
  108. data/cookbooks/firewall/recipes/default.rb +7 -7
  109. data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
  110. data/cookbooks/firewall/recipes/firewalld.rb +87 -0
  111. data/cookbooks/firewall/renovate.json +18 -0
  112. data/cookbooks/firewall/resources/firewalld.rb +28 -0
  113. data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
  114. data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
  115. data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
  116. data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
  117. data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
  118. data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
  119. data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
  120. data/cookbooks/firewall/resources/nftables.rb +71 -0
  121. data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
  122. data/cookbooks/mu-activedirectory/Berksfile +1 -1
  123. data/cookbooks/mu-activedirectory/metadata.rb +1 -1
  124. data/cookbooks/mu-firewall/metadata.rb +2 -2
  125. data/cookbooks/mu-master/Berksfile +4 -3
  126. data/cookbooks/mu-master/attributes/default.rb +5 -2
  127. data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
  128. data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
  129. data/cookbooks/mu-master/libraries/mu.rb +24 -0
  130. data/cookbooks/mu-master/metadata.rb +5 -5
  131. data/cookbooks/mu-master/recipes/default.rb +31 -20
  132. data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
  133. data/cookbooks/mu-master/recipes/init.rb +58 -19
  134. data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
  135. data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
  136. data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
  137. data/cookbooks/mu-php54/Berksfile +1 -1
  138. data/cookbooks/mu-php54/metadata.rb +2 -2
  139. data/cookbooks/mu-tools/Berksfile +2 -3
  140. data/cookbooks/mu-tools/attributes/default.rb +3 -4
  141. data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
  142. data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
  143. data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
  144. data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
  145. data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
  146. data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
  147. data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
  148. data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
  149. data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
  150. data/cookbooks/mu-tools/libraries/helper.rb +21 -9
  151. data/cookbooks/mu-tools/metadata.rb +4 -4
  152. data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
  153. data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
  154. data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
  155. data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
  156. data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
  157. data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
  158. data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
  159. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  160. data/data_bags/nagios_services/apm_backend_connect.json +5 -0
  161. data/data_bags/nagios_services/apm_listen.json +5 -0
  162. data/data_bags/nagios_services/elastic_shards.json +5 -0
  163. data/data_bags/nagios_services/logstash.json +5 -0
  164. data/data_bags/nagios_services/rhel7_updates.json +8 -0
  165. data/extras/image-generators/AWS/centos7.yaml +1 -0
  166. data/extras/image-generators/AWS/rhel7.yaml +21 -0
  167. data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
  168. data/extras/image-generators/AWS/win2k16.yaml +1 -0
  169. data/extras/image-generators/AWS/win2k19.yaml +1 -0
  170. data/extras/list-stock-amis +0 -0
  171. data/extras/ruby_rpm/muby.spec +8 -5
  172. data/extras/vault_tools/export_vaults.sh +1 -1
  173. data/extras/vault_tools/recreate_vaults.sh +0 -0
  174. data/extras/vault_tools/test_vaults.sh +0 -0
  175. data/install/deprecated-bash-library.sh +1 -1
  176. data/install/installer +4 -2
  177. data/modules/mommacat.ru +3 -1
  178. data/modules/mu/adoption.rb +1 -1
  179. data/modules/mu/cloud/dnszone.rb +2 -2
  180. data/modules/mu/cloud/machine_images.rb +26 -25
  181. data/modules/mu/cloud/resource_base.rb +213 -182
  182. data/modules/mu/cloud/server_pool.rb +1 -1
  183. data/modules/mu/cloud/ssh_sessions.rb +7 -5
  184. data/modules/mu/cloud/wrappers.rb +2 -2
  185. data/modules/mu/cloud.rb +1 -1
  186. data/modules/mu/config/bucket.rb +1 -1
  187. data/modules/mu/config/function.rb +6 -1
  188. data/modules/mu/config/loadbalancer.rb +24 -2
  189. data/modules/mu/config/ref.rb +12 -0
  190. data/modules/mu/config/role.rb +1 -1
  191. data/modules/mu/config/schema_helpers.rb +42 -9
  192. data/modules/mu/config/server.rb +43 -27
  193. data/modules/mu/config/tail.rb +19 -10
  194. data/modules/mu/config.rb +6 -5
  195. data/modules/mu/defaults/AWS.yaml +78 -114
  196. data/modules/mu/deploy.rb +9 -2
  197. data/modules/mu/groomer.rb +12 -4
  198. data/modules/mu/groomers/ansible.rb +104 -20
  199. data/modules/mu/groomers/chef.rb +15 -6
  200. data/modules/mu/master.rb +9 -4
  201. data/modules/mu/mommacat/daemon.rb +4 -2
  202. data/modules/mu/mommacat/naming.rb +1 -2
  203. data/modules/mu/mommacat/storage.rb +7 -2
  204. data/modules/mu/mommacat.rb +33 -6
  205. data/modules/mu/providers/aws/database.rb +161 -8
  206. data/modules/mu/providers/aws/dnszone.rb +11 -6
  207. data/modules/mu/providers/aws/endpoint.rb +81 -6
  208. data/modules/mu/providers/aws/firewall_rule.rb +254 -172
  209. data/modules/mu/providers/aws/function.rb +65 -3
  210. data/modules/mu/providers/aws/loadbalancer.rb +39 -28
  211. data/modules/mu/providers/aws/log.rb +2 -1
  212. data/modules/mu/providers/aws/role.rb +25 -7
  213. data/modules/mu/providers/aws/server.rb +36 -12
  214. data/modules/mu/providers/aws/server_pool.rb +237 -127
  215. data/modules/mu/providers/aws/storage_pool.rb +7 -1
  216. data/modules/mu/providers/aws/user.rb +1 -1
  217. data/modules/mu/providers/aws/userdata/linux.erb +6 -2
  218. data/modules/mu/providers/aws/userdata/windows.erb +7 -5
  219. data/modules/mu/providers/aws/vpc.rb +49 -25
  220. data/modules/mu/providers/aws.rb +13 -8
  221. data/modules/mu/providers/azure/container_cluster.rb +1 -1
  222. data/modules/mu/providers/azure/loadbalancer.rb +2 -2
  223. data/modules/mu/providers/azure/server.rb +5 -2
  224. data/modules/mu/providers/azure/userdata/linux.erb +1 -1
  225. data/modules/mu/providers/azure.rb +11 -8
  226. data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
  227. data/modules/mu/providers/google/container_cluster.rb +15 -2
  228. data/modules/mu/providers/google/folder.rb +2 -1
  229. data/modules/mu/providers/google/function.rb +130 -4
  230. data/modules/mu/providers/google/habitat.rb +2 -1
  231. data/modules/mu/providers/google/loadbalancer.rb +407 -160
  232. data/modules/mu/providers/google/role.rb +16 -3
  233. data/modules/mu/providers/google/server.rb +5 -1
  234. data/modules/mu/providers/google/user.rb +25 -18
  235. data/modules/mu/providers/google/userdata/linux.erb +1 -1
  236. data/modules/mu/providers/google/vpc.rb +53 -7
  237. data/modules/mu/providers/google.rb +39 -39
  238. data/modules/mu.rb +8 -8
  239. data/modules/tests/elk.yaml +46 -0
  240. data/test/mu-master-test/controls/all_in_one.rb +1 -1
  241. metadata +207 -112
  242. data/cookbooks/firewall/CONTRIBUTING.md +0 -2
  243. data/cookbooks/firewall/MAINTAINERS.md +0 -19
  244. data/cookbooks/firewall/libraries/matchers.rb +0 -30
  245. data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -564,7 +564,12 @@ module MU
564
564
  canned = Hash[MU::Cloud::Google.iam(credentials: args[:credentials]).list_roles.roles.map { |r| [r.name, r] }]
565
565
  begin
566
566
  MU::Cloud.resourceClass("Google", "Habitat").bindings(args[:project], credentials: args[:credentials]).each { |binding|
567
- found[binding.role] = canned[binding.role]
567
+ if binding.role =~ /^roles\//
568
+ found[binding.role] = canned[binding.role]
569
+ elsif binding.role =~ /^#{Regexp.quote(my_org.name)}\/roles\//
570
+ found[binding.role] = MU::Cloud::Google.iam(credentials: args[:credentials]).get_organization_role(binding.role)
571
+ end
572
+
568
573
  }
569
574
  rescue ::Google::Apis::ClientError => e
570
575
  raise e if !e.message.match(/forbidden: /)
@@ -663,7 +668,6 @@ module MU
663
668
  "credentials" => @config['credentials'],
664
669
  "cloud_id" => @cloud_id
665
670
  }
666
-
667
671
  my_org = MU::Cloud::Google.getOrg(@config['credentials'])
668
672
 
669
673
  # This can happen if the role_source isn't set correctly. This logic
@@ -680,6 +684,9 @@ module MU
680
684
 
681
685
  # GSuite or Cloud Identity role
682
686
  if cloud_desc.class == ::Google::Apis::AdminDirectoryV1::Role
687
+ if @cloud_id =~ /ncbi_snapshot_manager/
688
+ MU.log "directory-tier role", MU::NOTICE
689
+ end
683
690
  return nil if cloud_desc.is_system_role
684
691
 
685
692
  bok["name"] = @config['name'].gsub(/[^a-z0-9]/i, '-').downcase
@@ -702,6 +709,9 @@ module MU
702
709
  bok['import'].sort! # at least be legible
703
710
  end
704
711
  else # otherwise it's a GCP IAM role of some kind
712
+ if @cloud_id =~ /ncbi_snapshot_manager/
713
+ MU.log "cloud-tier role", MU::NOTICE
714
+ end
705
715
 
706
716
  return nil if cloud_desc.stage == "DISABLED"
707
717
  if cloud_desc.name.match(/^roles\/([^\/]+)$/)
@@ -811,6 +821,9 @@ module MU
811
821
  # to bother with them.
812
822
  if bok['role_source'] == "canned" and
813
823
  (bok['bindings'].nil? or bok['bindings'].empty?)
824
+ if @cloud_id =~ /ncbi_snapshot_manager/
825
+ MU.log "ditching at canned role check", MU::NOTICE
826
+ end
814
827
  return nil
815
828
  end
816
829
 
@@ -940,7 +953,7 @@ module MU
940
953
  }
941
954
  rescue ::Google::Apis::ClientError => e
942
955
  if e.message.match(/forbidden: /)
943
- MU.log "Do not have permissions to retrieve bindings in project #{project}, skipping", MU::WARN
956
+ MU.log "Do not have permissions to retrieve bindings in project #{project}, org #{MU::Cloud::Google.getOrg(credentials).display_name}, skipping", MU::WARN
944
957
  else
945
958
  raise e
946
959
  end
@@ -1109,7 +1109,11 @@ next if !create
1109
1109
  # @param size [String]: Size (in gb) of the new volume
1110
1110
  # @param type [String]: Cloud storage type of the volume, if applicable
1111
1111
  # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
1112
- def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
1112
+ def addVolume(dev: nil, size: 0, type: "pd-standard", delete_on_termination: false)
1113
+ if dev.nil? or size == 0
1114
+ raise MuError, "Must specify a device name and a size for addVolume"
1115
+ end
1116
+
1113
1117
  devname = dev.gsub(/.*?\/([^\/]+)$/, '\1')
1114
1118
  resname = MU::Cloud::Google.nameStr(@mu_name+"-"+devname)
1115
1119
  MU.log "Creating disk #{resname}"
@@ -114,7 +114,7 @@ module MU
114
114
  primary_email: @config['email'],
115
115
  suspended: @config['suspend'],
116
116
  is_admin: @config['admin'],
117
- password: MU.generateWindowsPassword,
117
+ password: MU.generatePassword,
118
118
  change_password_at_next_login: (@config.has_key?('force_password_change') ? @config['force_password_change'] : true)
119
119
  )
120
120
 
@@ -310,6 +310,10 @@ module MU
310
310
  end
311
311
  end
312
312
 
313
+ # clock projects we can't list so we don't waste time trying them over
314
+ # and over
315
+ @@cant_list = []
316
+
313
317
  # Locate and return cloud provider descriptors of this resource type
314
318
  # which match the provided parameters, or all visible resources if no
315
319
  # filters are specified. At minimum, implementations of +find+ must
@@ -341,24 +345,27 @@ module MU
341
345
 
342
346
  if args[:project]
343
347
  # project-local service accounts
344
- resp = begin
345
- MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
346
- "projects/"+args[:project]
347
- )
348
- rescue ::Google::Apis::ClientError
349
- MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
350
- end
348
+ if !@@cant_list.include?(args[:project])
349
+ resp = begin
350
+ MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
351
+ "projects/"+args[:project]
352
+ )
353
+ rescue ::Google::Apis::ClientError
354
+ @@cant_list << args[:project]
355
+ MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}, org #{MU::Cloud::Google.getOrg(args[:credentials]).display_name}", MU::WARN
356
+ end
351
357
 
352
- if resp and resp.accounts
353
- resp.accounts.each { |sa|
354
- if args[:flags] and args[:flags]["skip_provider_owned"] and
355
- MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
356
- next
357
- end
358
- if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
359
- found[sa.name] = sa
360
- end
361
- }
358
+ if resp and resp.accounts
359
+ resp.accounts.each { |sa|
360
+ if args[:flags] and args[:flags]["skip_provider_owned"] and
361
+ MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
362
+ next
363
+ end
364
+ if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
365
+ found[sa.name] = sa
366
+ end
367
+ }
368
+ end
362
369
  end
363
370
  else
364
371
  if cred_cfg['masquerade_as']
@@ -105,7 +105,7 @@ umask 0077
105
105
 
106
106
  # Install Chef now, because why not?
107
107
  if [ ! -f /opt/chef/embedded/bin/ruby ];then
108
- curl https://www.chef.io/chef/install.sh > chef-install.sh
108
+ curl https://omnitruck.chef.io/install.sh > chef-install.sh
109
109
  set +e
110
110
  # We may run afoul of a synchronous bootstrap process doing the same thing. So
111
111
  # wait until we've managed to run successfully.
@@ -457,7 +457,7 @@ end
457
457
  end
458
458
 
459
459
  if name
460
- subnet_mu_name ||= @config['scrub_mu_isms'] ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
460
+ subnet_mu_name ||= (@config['scrub_mu_isms'] or !@deploy) ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
461
461
  end
462
462
 
463
463
  MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region}, subnet_mu_name: #{subnet_mu_name})", MU::DEBUG, details: caller[0]
@@ -976,12 +976,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
976
976
  # @return [String]
977
977
  def getUnusedAddressBlock(exclude: [], max_bits: 28)
978
978
  used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
979
- subnets.each { |s|
980
- used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
981
- if s.cloud_desc.secondary_ip_ranges
982
- used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
983
- end
984
- }
979
+ used_ranges.concat(listSubnetRanges)
985
980
  # XXX sort used_ranges
986
981
  candidate = used_ranges.first.next_sib
987
982
 
@@ -1003,8 +998,55 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1003
998
  candidate.to_s
1004
999
  end
1005
1000
 
1001
+ # Add a new secondary IP range to the given subnet, if it doesn't
1002
+ # already exist
1003
+ def addSecondaryRange(subnet, cidr, name)
1004
+ subnet = getSubnet(cloud_id: subnet, name: subnet, subnet_mu_name: subnet)
1005
+ if !subnet
1006
+ raise MuError, "#{self.to_s} failed to locate a subnet from '#{subnet}'"
1007
+ end
1008
+
1009
+ secondary_ranges = subnet.cloud_desc.secondary_ip_ranges
1010
+ secondary_ranges ||= []
1011
+ secondary_ranges.each { |r|
1012
+ if r.ip_cidr_range == cidr and r.range_name == name
1013
+ return
1014
+ elsif r.ip_cidr_range == cidr or r.range_name == name
1015
+ MU.log "Conflicting secondary IP range, cannot add #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}", MU::WARN, details: r
1016
+ return
1017
+ end
1018
+ }
1019
+
1020
+ secondary_ranges << MU::Cloud::Google.compute(:SubnetworkSecondaryRange).new(
1021
+ ip_cidr_range: cidr,
1022
+ range_name: name
1023
+ )
1024
+ MU.log "Adding new secondary IP range #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}"
1025
+ subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
1026
+ name: subnet.cloud_desc.name,
1027
+ secondary_ip_ranges: secondary_ranges,
1028
+ fingerprint: subnet.cloud_desc.fingerprint
1029
+ )
1030
+ MU::Cloud::Google.compute(credentials: @credentials).patch_subnetwork(@project_id, subnet.az, subnet.cloud_desc.name, subnetobj)
1031
+ end
1032
+
1033
+ def connector(id: nil, name: nil)
1034
+ end
1035
+
1006
1036
  private
1007
1037
 
1038
+ # @return [Array<NetAddr::IPv4Net>]
1039
+ def listSubnetRanges
1040
+ ranges = []
1041
+ subnets.each { |s|
1042
+ ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
1043
+ if s.cloud_desc.secondary_ip_ranges
1044
+ ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
1045
+ end
1046
+ }
1047
+ ranges
1048
+ end
1049
+
1008
1050
  def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
1009
1051
  private_acl = {
1010
1052
  "name" => vpc_name+"-rt",
@@ -1209,6 +1251,10 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1209
1251
  # @return [Boolean]
1210
1252
  def private?
1211
1253
  @parent.cloud_desc
1254
+ if !@parent.routes
1255
+ MU.log "Failed to retrieve routes from #{@parent.to_s}", MU::WARN
1256
+ return true
1257
+ end
1212
1258
  @parent.routes.map { |r|
1213
1259
  if r.dest_range == "0.0.0.0/0" and !r.next_hop_gateway.nil? and
1214
1260
  (r.tags.nil? or r.tags.size == 0) and
@@ -29,8 +29,6 @@ module MU
29
29
  @@authorizers = {}
30
30
  @@acct_to_profile_map = {}
31
31
  @@enable_semaphores = {}
32
- @@readonly_semaphore = Mutex.new
33
- @@readonly = {}
34
32
 
35
33
  # Module used by {MU::Cloud} to insert additional instance methods into
36
34
  # instantiated resources in this cloud layer.
@@ -40,7 +38,7 @@ module MU
40
38
  # @return [String]
41
39
  def url
42
40
  desc = cloud_desc
43
- (desc and desc.self_link) ? desc.self_link : nil
41
+ (desc and desc.respond_to?(:self_link) and desc.self_link) ? desc.self_link : nil
44
42
  end
45
43
  end
46
44
 
@@ -52,6 +50,12 @@ module MU
52
50
  [:url]
53
51
  end
54
52
 
53
+ # Cloud-specific resource methods or attributes we want exposed for
54
+ # reading by {MU::Cloud}
55
+ def self.customAttrReaders
56
+ [:project_id, :customer]
57
+ end
58
+
55
59
  # Is this a "real" cloud provider, or a stub like CloudFormation?
56
60
  def self.virtual?
57
61
  false
@@ -75,12 +79,6 @@ module MU
75
79
  # @param cloudobj [MU::Cloud]
76
80
  # @param deploy [MU::MommaCat]
77
81
  def self.resourceInitHook(cloudobj, deploy)
78
- class << self
79
- attr_reader :project_id
80
- attr_reader :customer
81
- # url is too complex for an attribute (we get it from the cloud API),
82
- # so it's up in AdditionalResourceMethods instead
83
- end
84
82
  return if !cloudobj
85
83
 
86
84
  cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials']))
@@ -96,9 +94,6 @@ module MU
96
94
  cloudobj.instance_variable_set(:@project_id, cloudobj.config['project'])
97
95
  end
98
96
 
99
- # XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe
100
- # that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end
101
-
102
97
  # XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
103
98
  end
104
99
 
@@ -346,8 +341,10 @@ module MU
346
341
  end
347
342
 
348
343
  # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
349
- # @param deploy_id [String]: The deploy for which we're writing the secret
344
+ # @param deploy [String]: The deploy for which we're writing the secret
350
345
  # @param value [String]: The contents of the secret
346
+ # @param name [String]: File/object name
347
+ # @param credentials [String]
351
348
  def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
352
349
  deploy_id = deploy.deploy_id
353
350
  name ||= deploy_id+"-secret"
@@ -847,7 +844,7 @@ MU.log e.message, MU::WARN, details: e.inspect
847
844
  require 'google/apis/iam_v1'
848
845
 
849
846
  if subclass.nil?
850
- @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials)
847
+ @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, retry_readonly: true)
851
848
  return @@iam_api[credentials]
852
849
  elsif subclass.is_a?(Symbol)
853
850
  return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
@@ -863,28 +860,12 @@ MU.log e.message, MU::WARN, details: e.inspect
863
860
  # dopey extra warnings about falling back on scopes
864
861
  credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)
865
862
 
866
- writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema']
867
- readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly']
868
- @@readonly_semaphore.synchronize {
869
- use_scopes = readscopes+writescopes
870
- if @@readonly[credentials] and @@readonly[credentials]["AdminDirectoryV1"]
871
- use_scopes = readscopes.dup
872
- end
873
-
874
- if subclass.nil?
875
- begin
876
- @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
877
- rescue Signet::AuthorizationError
878
- MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
879
- @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
880
- @@readonly[credentials] ||= {}
881
- @@readonly[credentials]["AdminDirectoryV1"] = true
882
- end
883
- return @@admin_directory_api[credentials]
884
- elsif subclass.is_a?(Symbol)
885
- return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
886
- end
887
- }
863
+ if subclass.nil?
864
+ @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true, retry_readonly: true)
865
+ return @@admin_directory_api[credentials]
866
+ elsif subclass.is_a?(Symbol)
867
+ return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
868
+ end
888
869
  end
889
870
 
890
871
  # Google's Cloud Resource Manager API
@@ -896,7 +877,7 @@ MU.log e.message, MU::WARN, details: e.inspect
896
877
  if !MU::Cloud::Google.credConfig(credentials)
897
878
  raise MuError, "No such credential set #{credentials} defined in mu.yaml!"
898
879
  end
899
- @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
880
+ @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], retry_readonly: true)
900
881
  return @@resource_api[credentials]
901
882
  elsif subclass.is_a?(Symbol)
902
883
  return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
@@ -1110,7 +1091,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1110
1091
  # Create a Google Cloud Platform API client
1111
1092
  # @param api [String]: Which API are we wrapping?
1112
1093
  # @param scopes [Array<String>]: Google auth scopes applicable to this API
1113
- def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
1094
+ def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false, retry_readonly: false)
1114
1095
  @credentials = credentials
1115
1096
  @scopes = scopes.map { |s|
1116
1097
  if !s.match(/\//) # allow callers to use shorthand
@@ -1127,6 +1108,22 @@ MU.log e.message, MU::WARN, details: e.inspect
1127
1108
  @api.authorization.sub = @masquerade
1128
1109
  @api.authorization.fetch_access_token!
1129
1110
  rescue Signet::AuthorizationError => e
1111
+ if retry_readonly
1112
+ newscopes = @scopes.map { |s|
1113
+ if s =~ /\/cloud-platform$/
1114
+ s += ".read-only"
1115
+ elsif s !~ /\/cloud-platform\b/ and s !~ /\.readonly$/
1116
+ s += ".readonly"
1117
+ end
1118
+ s
1119
+ }
1120
+ if newscopes != @scopes
1121
+ MU.log "Falling back to read-only access to #{api} API on credential set '#{credentials}'", MU::WARN, details: @scopes
1122
+ @scopes = newscopes
1123
+ @api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials)
1124
+ retry
1125
+ end
1126
+ end
1130
1127
  if auth_error_quiet
1131
1128
  MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
1132
1129
  else
@@ -1281,7 +1278,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1281
1278
  raise e
1282
1279
  end
1283
1280
  rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
1284
- if e.message.match(/^quotaExceeded: Request rate/)
1281
+ if e.message.match(/^quotaExceeded: Request rate|failedPrecondition.*?already in progress/)
1285
1282
  if retries <= 10
1286
1283
  sleep wait_backoff
1287
1284
  retries += 1
@@ -1457,6 +1454,9 @@ MU.log e.message, MU::WARN, details: e.inspect
1457
1454
  faked_args.pop
1458
1455
  faked_args.pop
1459
1456
  end
1457
+ if method_sym == :patch_subnetwork
1458
+ faked_args.pop
1459
+ end
1460
1460
  faked_args.push(cloud_id)
1461
1461
  if get_method == :get_project_location_cluster
1462
1462
  faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
data/modules/mu.rb CHANGED
@@ -701,7 +701,7 @@ module MU
701
701
  !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
702
702
  @@mu_public_addr = $MU_CFG['public_address']
703
703
  if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/) and
704
- File.exists?("/etc/hostname") and File.exists?("/etc/hosts")
704
+ File.exist?("/etc/hostname") and File.exist?("/etc/hosts")
705
705
  hostname = IO.readlines("/etc/hostname")[0].gsub(/\n/, '')
706
706
 
707
707
  hostlines = File.open('/etc/hosts').grep(/.*#{hostname}.*/)
@@ -788,7 +788,7 @@ module MU
788
788
  end
789
789
 
790
790
  # The version of Chef we will install on nodes.
791
- @@chefVersion = "14.0.190"
791
+ @@chefVersion = "18.5.0"
792
792
  # The version of Chef we will install on nodes.
793
793
  # @return [String]
794
794
  def self.chefVersion
@@ -1072,23 +1072,23 @@ module MU
1072
1072
  end
1073
1073
 
1074
1074
 
1075
- # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
1075
+ # Generate a random password which will satisfy the complexity requirements of things like the stock Amazon Windows AMIs.
1076
1076
  # return [String]: A password string.
1077
- def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 50)
1077
+ def self.generatePassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', length: 14, retries: 50)
1078
1078
  # We have dopey complexity requirements, be stringent here.
1079
1079
  # I'll be nice and not condense this into one elegant-but-unreadable regular expression
1080
1080
  attempts = 0
1081
1081
  safe_metachars = Regexp.escape(safe_pattern)
1082
1082
  begin
1083
1083
  if attempts > retries
1084
- MU.log "Failed to generate an adequate Windows password after #{attempts} attempts", MU::ERR
1085
- raise MuError, "Failed to generate an adequate Windows password after #{attempts} attempts"
1084
+ MU.log "Failed to generate an adequate password after #{attempts} attempts", MU::ERR
1085
+ raise MuError, "Failed to generate an adequate password after #{attempts} attempts"
1086
1086
  end
1087
- winpass = Password.random(14..16)
1087
+ winpass = Password.random(length..(length+2))
1088
1088
  attempts += 1
1089
1089
  end while winpass.nil? or !winpass.match(/^[a-z]/i) or !winpass.match(/[A-Z]/) or !winpass.match(/[a-z]/) or !winpass.match(/\d/) or !winpass.match(/[#{safe_metachars}]/) or winpass.match(/[^\w\d#{safe_metachars}]/)
1090
1090
 
1091
- MU.log "Generated Windows password after #{attempts} attempts", MU::DEBUG
1091
+ MU.log "Generated password after #{attempts} attempts", MU::DEBUG
1092
1092
  return winpass
1093
1093
  end
1094
1094
 
@@ -0,0 +1,46 @@
1
+ # groomers: Ansible
2
+ ---
3
+ appname: smoketest
4
+ generate_passwords:
5
+ - itemname: elasticpw
6
+ minlength: 12
7
+ vpcs:
8
+ - name: wrapper
9
+ servers:
10
+ - name: frontend
11
+ platform: centos7
12
+ groomer: Ansible
13
+ vpc:
14
+ name: wrapper
15
+ size: t3.medium
16
+ vault_access:
17
+ - item: elasticpw
18
+ run_list:
19
+ - mu-logstash
20
+ #<% if cloud != "AWS" %>
21
+ - name: backend
22
+ platform: centos7
23
+ groomer: Ansible
24
+ vpc:
25
+ name: wrapper
26
+ size: m5.large
27
+ vault_access:
28
+ - item: elasticpw
29
+ run_list:
30
+ - mu-elastic
31
+ #<% else %>
32
+ #search_domains:
33
+ #- name: logsearch
34
+ # elasticsearch_version: '7.4'
35
+ # instance_count: 1
36
+ # instance_type: r5.large.elasticsearch
37
+ # ebs_size: 10
38
+ # ebs_type: gp2
39
+ # access_policies:
40
+ # Version: '2012-10-17'
41
+ # Statement:
42
+ # - Effect: Allow
43
+ # Principal:
44
+ # AWS: "*"
45
+ # Action: es:ESHttp*
46
+ #<% end %>
@@ -15,7 +15,7 @@ control 'init' do
15
15
  node = json('/tmp/chef_node.json').params
16
16
  NODE_PUB_IP=node_meta[0]['pub_ip']
17
17
  CHEF_SERVER_VERSION="12.17.15-1"
18
- CHEF_CLIENT_VERSION="14.0.190"
18
+ CHEF_CLIENT_VERSION="18.2.7"
19
19
  KNIFE_WINDOWS="1.9.0"
20
20
  MU_BASE="/opt/mu"
21
21
  f = "/etc/ssh/sshd_config"