cloud-mu 3.5.0 → 3.6.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +5 -2
  3. data/Berksfile.lock +135 -0
  4. data/ansible/roles/mu-base/README.md +33 -0
  5. data/ansible/roles/mu-base/defaults/main.yml +2 -0
  6. data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
  7. data/ansible/roles/mu-base/files/check_apm.sh +18 -0
  8. data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
  9. data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
  10. data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
  11. data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
  12. data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
  13. data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
  14. data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
  15. data/ansible/roles/mu-base/files/logrotate.conf +35 -0
  16. data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
  17. data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
  18. data/ansible/roles/mu-base/handlers/main.yml +5 -0
  19. data/ansible/roles/mu-base/meta/main.yml +53 -0
  20. data/ansible/roles/mu-base/tasks/main.yml +113 -0
  21. data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
  22. data/ansible/roles/mu-base/tests/inventory +2 -0
  23. data/ansible/roles/mu-base/tests/test.yml +5 -0
  24. data/ansible/roles/mu-base/vars/main.yml +1 -0
  25. data/ansible/roles/mu-compliance/README.md +33 -0
  26. data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
  27. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
  28. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
  29. data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
  30. data/ansible/roles/mu-compliance/meta/main.yml +53 -0
  31. data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
  32. data/ansible/roles/mu-compliance/tests/inventory +2 -0
  33. data/ansible/roles/mu-compliance/tests/test.yml +5 -0
  34. data/ansible/roles/mu-compliance/vars/main.yml +4 -0
  35. data/ansible/roles/mu-elastic/README.md +51 -0
  36. data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
  37. data/ansible/roles/mu-elastic/files/jvm.options +93 -0
  38. data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
  39. data/ansible/roles/mu-elastic/meta/main.yml +52 -0
  40. data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
  41. data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
  42. data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
  43. data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
  44. data/ansible/roles/mu-elastic/tests/inventory +2 -0
  45. data/ansible/roles/mu-elastic/tests/test.yml +5 -0
  46. data/ansible/roles/mu-elastic/vars/main.yml +2 -0
  47. data/ansible/roles/mu-logstash/README.md +51 -0
  48. data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
  49. data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
  50. data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
  51. data/ansible/roles/mu-logstash/files/jvm.options +84 -0
  52. data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
  53. data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
  54. data/ansible/roles/mu-logstash/meta/main.yml +52 -0
  55. data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
  56. data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
  57. data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
  58. data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
  59. data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
  60. data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
  61. data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
  62. data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
  63. data/ansible/roles/mu-logstash/tests/inventory +2 -0
  64. data/ansible/roles/mu-logstash/tests/test.yml +5 -0
  65. data/ansible/roles/mu-logstash/vars/main.yml +2 -0
  66. data/ansible/roles/mu-rdp/README.md +33 -0
  67. data/ansible/roles/mu-rdp/meta/main.yml +53 -0
  68. data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
  69. data/ansible/roles/mu-rdp/tests/inventory +2 -0
  70. data/ansible/roles/mu-rdp/tests/test.yml +5 -0
  71. data/ansible/roles/mu-windows/tasks/main.yml +3 -0
  72. data/bin/mu-ansible-secret +1 -1
  73. data/bin/mu-aws-setup +4 -3
  74. data/bin/mu-azure-setup +5 -5
  75. data/bin/mu-configure +25 -17
  76. data/bin/mu-firewall-allow-clients +1 -0
  77. data/bin/mu-gcp-setup +3 -3
  78. data/bin/mu-load-config.rb +1 -0
  79. data/bin/mu-node-manage +66 -33
  80. data/bin/mu-self-update +2 -2
  81. data/bin/mu-upload-chef-artifacts +6 -1
  82. data/bin/mu-user-manage +1 -1
  83. data/cloud-mu.gemspec +25 -23
  84. data/cookbooks/firewall/CHANGELOG.md +417 -224
  85. data/cookbooks/firewall/LICENSE +202 -0
  86. data/cookbooks/firewall/README.md +153 -126
  87. data/cookbooks/firewall/TODO.md +6 -0
  88. data/cookbooks/firewall/attributes/firewalld.rb +7 -0
  89. data/cookbooks/firewall/attributes/iptables.rb +3 -3
  90. data/cookbooks/firewall/chefignore +115 -0
  91. data/cookbooks/firewall/libraries/helpers.rb +5 -0
  92. data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
  93. data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
  94. data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
  95. data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
  96. data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
  97. data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
  98. data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
  99. data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
  100. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
  101. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
  102. data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
  103. data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
  104. data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
  105. data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
  106. data/cookbooks/firewall/metadata.json +40 -1
  107. data/cookbooks/firewall/metadata.rb +15 -0
  108. data/cookbooks/firewall/recipes/default.rb +7 -7
  109. data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
  110. data/cookbooks/firewall/recipes/firewalld.rb +87 -0
  111. data/cookbooks/firewall/renovate.json +18 -0
  112. data/cookbooks/firewall/resources/firewalld.rb +28 -0
  113. data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
  114. data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
  115. data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
  116. data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
  117. data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
  118. data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
  119. data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
  120. data/cookbooks/firewall/resources/nftables.rb +71 -0
  121. data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
  122. data/cookbooks/mu-activedirectory/Berksfile +1 -1
  123. data/cookbooks/mu-activedirectory/metadata.rb +1 -1
  124. data/cookbooks/mu-firewall/metadata.rb +2 -2
  125. data/cookbooks/mu-master/Berksfile +4 -3
  126. data/cookbooks/mu-master/attributes/default.rb +5 -2
  127. data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
  128. data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
  129. data/cookbooks/mu-master/libraries/mu.rb +24 -0
  130. data/cookbooks/mu-master/metadata.rb +5 -5
  131. data/cookbooks/mu-master/recipes/default.rb +31 -20
  132. data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
  133. data/cookbooks/mu-master/recipes/init.rb +58 -19
  134. data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
  135. data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
  136. data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
  137. data/cookbooks/mu-php54/Berksfile +1 -1
  138. data/cookbooks/mu-php54/metadata.rb +2 -2
  139. data/cookbooks/mu-tools/Berksfile +2 -3
  140. data/cookbooks/mu-tools/attributes/default.rb +3 -4
  141. data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
  142. data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
  143. data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
  144. data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
  145. data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
  146. data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
  147. data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
  148. data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
  149. data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
  150. data/cookbooks/mu-tools/libraries/helper.rb +21 -9
  151. data/cookbooks/mu-tools/metadata.rb +4 -4
  152. data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
  153. data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
  154. data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
  155. data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
  156. data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
  157. data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
  158. data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
  159. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  160. data/data_bags/nagios_services/apm_backend_connect.json +5 -0
  161. data/data_bags/nagios_services/apm_listen.json +5 -0
  162. data/data_bags/nagios_services/elastic_shards.json +5 -0
  163. data/data_bags/nagios_services/logstash.json +5 -0
  164. data/data_bags/nagios_services/rhel7_updates.json +8 -0
  165. data/extras/image-generators/AWS/centos7.yaml +1 -0
  166. data/extras/image-generators/AWS/rhel7.yaml +21 -0
  167. data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
  168. data/extras/image-generators/AWS/win2k16.yaml +1 -0
  169. data/extras/image-generators/AWS/win2k19.yaml +1 -0
  170. data/extras/list-stock-amis +0 -0
  171. data/extras/ruby_rpm/muby.spec +8 -5
  172. data/extras/vault_tools/export_vaults.sh +1 -1
  173. data/extras/vault_tools/recreate_vaults.sh +0 -0
  174. data/extras/vault_tools/test_vaults.sh +0 -0
  175. data/install/deprecated-bash-library.sh +1 -1
  176. data/install/installer +4 -2
  177. data/modules/mommacat.ru +3 -1
  178. data/modules/mu/adoption.rb +1 -1
  179. data/modules/mu/cloud/dnszone.rb +2 -2
  180. data/modules/mu/cloud/machine_images.rb +26 -25
  181. data/modules/mu/cloud/resource_base.rb +213 -182
  182. data/modules/mu/cloud/server_pool.rb +1 -1
  183. data/modules/mu/cloud/ssh_sessions.rb +7 -5
  184. data/modules/mu/cloud/wrappers.rb +2 -2
  185. data/modules/mu/cloud.rb +1 -1
  186. data/modules/mu/config/bucket.rb +1 -1
  187. data/modules/mu/config/function.rb +6 -1
  188. data/modules/mu/config/loadbalancer.rb +24 -2
  189. data/modules/mu/config/ref.rb +12 -0
  190. data/modules/mu/config/role.rb +1 -1
  191. data/modules/mu/config/schema_helpers.rb +42 -9
  192. data/modules/mu/config/server.rb +43 -27
  193. data/modules/mu/config/tail.rb +19 -10
  194. data/modules/mu/config.rb +6 -5
  195. data/modules/mu/defaults/AWS.yaml +78 -114
  196. data/modules/mu/deploy.rb +9 -2
  197. data/modules/mu/groomer.rb +12 -4
  198. data/modules/mu/groomers/ansible.rb +104 -20
  199. data/modules/mu/groomers/chef.rb +15 -6
  200. data/modules/mu/master.rb +9 -4
  201. data/modules/mu/mommacat/daemon.rb +4 -2
  202. data/modules/mu/mommacat/naming.rb +1 -2
  203. data/modules/mu/mommacat/storage.rb +7 -2
  204. data/modules/mu/mommacat.rb +33 -6
  205. data/modules/mu/providers/aws/database.rb +161 -8
  206. data/modules/mu/providers/aws/dnszone.rb +11 -6
  207. data/modules/mu/providers/aws/endpoint.rb +81 -6
  208. data/modules/mu/providers/aws/firewall_rule.rb +254 -172
  209. data/modules/mu/providers/aws/function.rb +65 -3
  210. data/modules/mu/providers/aws/loadbalancer.rb +39 -28
  211. data/modules/mu/providers/aws/log.rb +2 -1
  212. data/modules/mu/providers/aws/role.rb +25 -7
  213. data/modules/mu/providers/aws/server.rb +36 -12
  214. data/modules/mu/providers/aws/server_pool.rb +237 -127
  215. data/modules/mu/providers/aws/storage_pool.rb +7 -1
  216. data/modules/mu/providers/aws/user.rb +1 -1
  217. data/modules/mu/providers/aws/userdata/linux.erb +6 -2
  218. data/modules/mu/providers/aws/userdata/windows.erb +7 -5
  219. data/modules/mu/providers/aws/vpc.rb +49 -25
  220. data/modules/mu/providers/aws.rb +13 -8
  221. data/modules/mu/providers/azure/container_cluster.rb +1 -1
  222. data/modules/mu/providers/azure/loadbalancer.rb +2 -2
  223. data/modules/mu/providers/azure/server.rb +5 -2
  224. data/modules/mu/providers/azure/userdata/linux.erb +1 -1
  225. data/modules/mu/providers/azure.rb +11 -8
  226. data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
  227. data/modules/mu/providers/google/container_cluster.rb +15 -2
  228. data/modules/mu/providers/google/folder.rb +2 -1
  229. data/modules/mu/providers/google/function.rb +130 -4
  230. data/modules/mu/providers/google/habitat.rb +2 -1
  231. data/modules/mu/providers/google/loadbalancer.rb +407 -160
  232. data/modules/mu/providers/google/role.rb +16 -3
  233. data/modules/mu/providers/google/server.rb +5 -1
  234. data/modules/mu/providers/google/user.rb +25 -18
  235. data/modules/mu/providers/google/userdata/linux.erb +1 -1
  236. data/modules/mu/providers/google/vpc.rb +53 -7
  237. data/modules/mu/providers/google.rb +39 -39
  238. data/modules/mu.rb +8 -8
  239. data/modules/tests/elk.yaml +46 -0
  240. data/test/mu-master-test/controls/all_in_one.rb +1 -1
  241. metadata +207 -112
  242. data/cookbooks/firewall/CONTRIBUTING.md +0 -2
  243. data/cookbooks/firewall/MAINTAINERS.md +0 -19
  244. data/cookbooks/firewall/libraries/matchers.rb +0 -30
  245. data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -564,7 +564,12 @@ module MU
564
564
  canned = Hash[MU::Cloud::Google.iam(credentials: args[:credentials]).list_roles.roles.map { |r| [r.name, r] }]
565
565
  begin
566
566
  MU::Cloud.resourceClass("Google", "Habitat").bindings(args[:project], credentials: args[:credentials]).each { |binding|
567
- found[binding.role] = canned[binding.role]
567
+ if binding.role =~ /^roles\//
568
+ found[binding.role] = canned[binding.role]
569
+ elsif binding.role =~ /^#{Regexp.quote(my_org.name)}\/roles\//
570
+ found[binding.role] = MU::Cloud::Google.iam(credentials: args[:credentials]).get_organization_role(binding.role)
571
+ end
572
+
568
573
  }
569
574
  rescue ::Google::Apis::ClientError => e
570
575
  raise e if !e.message.match(/forbidden: /)
@@ -663,7 +668,6 @@ module MU
663
668
  "credentials" => @config['credentials'],
664
669
  "cloud_id" => @cloud_id
665
670
  }
666
-
667
671
  my_org = MU::Cloud::Google.getOrg(@config['credentials'])
668
672
 
669
673
  # This can happen if the role_source isn't set correctly. This logic
@@ -680,6 +684,9 @@ module MU
680
684
 
681
685
  # GSuite or Cloud Identity role
682
686
  if cloud_desc.class == ::Google::Apis::AdminDirectoryV1::Role
687
+ if @cloud_id =~ /ncbi_snapshot_manager/
688
+ MU.log "directory-tier role", MU::NOTICE
689
+ end
683
690
  return nil if cloud_desc.is_system_role
684
691
 
685
692
  bok["name"] = @config['name'].gsub(/[^a-z0-9]/i, '-').downcase
@@ -702,6 +709,9 @@ module MU
702
709
  bok['import'].sort! # at least be legible
703
710
  end
704
711
  else # otherwise it's a GCP IAM role of some kind
712
+ if @cloud_id =~ /ncbi_snapshot_manager/
713
+ MU.log "cloud-tier role", MU::NOTICE
714
+ end
705
715
 
706
716
  return nil if cloud_desc.stage == "DISABLED"
707
717
  if cloud_desc.name.match(/^roles\/([^\/]+)$/)
@@ -811,6 +821,9 @@ module MU
811
821
  # to bother with them.
812
822
  if bok['role_source'] == "canned" and
813
823
  (bok['bindings'].nil? or bok['bindings'].empty?)
824
+ if @cloud_id =~ /ncbi_snapshot_manager/
825
+ MU.log "ditching at canned role check", MU::NOTICE
826
+ end
814
827
  return nil
815
828
  end
816
829
 
@@ -940,7 +953,7 @@ module MU
940
953
  }
941
954
  rescue ::Google::Apis::ClientError => e
942
955
  if e.message.match(/forbidden: /)
943
- MU.log "Do not have permissions to retrieve bindings in project #{project}, skipping", MU::WARN
956
+ MU.log "Do not have permissions to retrieve bindings in project #{project}, org #{MU::Cloud::Google.getOrg(credentials).display_name}, skipping", MU::WARN
944
957
  else
945
958
  raise e
946
959
  end
@@ -1109,7 +1109,11 @@ next if !create
1109
1109
  # @param size [String]: Size (in gb) of the new volume
1110
1110
  # @param type [String]: Cloud storage type of the volume, if applicable
1111
1111
  # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
1112
- def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
1112
+ def addVolume(dev: nil, size: 0, type: "pd-standard", delete_on_termination: false)
1113
+ if dev.nil? or size == 0
1114
+ raise MuError, "Must specify a device name and a size for addVolume"
1115
+ end
1116
+
1113
1117
  devname = dev.gsub(/.*?\/([^\/]+)$/, '\1')
1114
1118
  resname = MU::Cloud::Google.nameStr(@mu_name+"-"+devname)
1115
1119
  MU.log "Creating disk #{resname}"
@@ -114,7 +114,7 @@ module MU
114
114
  primary_email: @config['email'],
115
115
  suspended: @config['suspend'],
116
116
  is_admin: @config['admin'],
117
- password: MU.generateWindowsPassword,
117
+ password: MU.generatePassword,
118
118
  change_password_at_next_login: (@config.has_key?('force_password_change') ? @config['force_password_change'] : true)
119
119
  )
120
120
 
@@ -310,6 +310,10 @@ module MU
310
310
  end
311
311
  end
312
312
 
313
+ # clock projects we can't list so we don't waste time trying them over
314
+ # and over
315
+ @@cant_list = []
316
+
313
317
  # Locate and return cloud provider descriptors of this resource type
314
318
  # which match the provided parameters, or all visible resources if no
315
319
  # filters are specified. At minimum, implementations of +find+ must
@@ -341,24 +345,27 @@ module MU
341
345
 
342
346
  if args[:project]
343
347
  # project-local service accounts
344
- resp = begin
345
- MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
346
- "projects/"+args[:project]
347
- )
348
- rescue ::Google::Apis::ClientError
349
- MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
350
- end
348
+ if !@@cant_list.include?(args[:project])
349
+ resp = begin
350
+ MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
351
+ "projects/"+args[:project]
352
+ )
353
+ rescue ::Google::Apis::ClientError
354
+ @@cant_list << args[:project]
355
+ MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}, org #{MU::Cloud::Google.getOrg(args[:credentials]).display_name}", MU::WARN
356
+ end
351
357
 
352
- if resp and resp.accounts
353
- resp.accounts.each { |sa|
354
- if args[:flags] and args[:flags]["skip_provider_owned"] and
355
- MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
356
- next
357
- end
358
- if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
359
- found[sa.name] = sa
360
- end
361
- }
358
+ if resp and resp.accounts
359
+ resp.accounts.each { |sa|
360
+ if args[:flags] and args[:flags]["skip_provider_owned"] and
361
+ MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
362
+ next
363
+ end
364
+ if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
365
+ found[sa.name] = sa
366
+ end
367
+ }
368
+ end
362
369
  end
363
370
  else
364
371
  if cred_cfg['masquerade_as']
@@ -105,7 +105,7 @@ umask 0077
105
105
 
106
106
  # Install Chef now, because why not?
107
107
  if [ ! -f /opt/chef/embedded/bin/ruby ];then
108
- curl https://www.chef.io/chef/install.sh > chef-install.sh
108
+ curl https://omnitruck.chef.io/install.sh > chef-install.sh
109
109
  set +e
110
110
  # We may run afoul of a synchronous bootstrap process doing the same thing. So
111
111
  # wait until we've managed to run successfully.
@@ -457,7 +457,7 @@ end
457
457
  end
458
458
 
459
459
  if name
460
- subnet_mu_name ||= @config['scrub_mu_isms'] ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
460
+ subnet_mu_name ||= (@config['scrub_mu_isms'] or !@deploy) ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
461
461
  end
462
462
 
463
463
  MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region}, subnet_mu_name: #{subnet_mu_name})", MU::DEBUG, details: caller[0]
@@ -976,12 +976,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
976
976
  # @return [String]
977
977
  def getUnusedAddressBlock(exclude: [], max_bits: 28)
978
978
  used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
979
- subnets.each { |s|
980
- used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
981
- if s.cloud_desc.secondary_ip_ranges
982
- used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
983
- end
984
- }
979
+ used_ranges.concat(listSubnetRanges)
985
980
  # XXX sort used_ranges
986
981
  candidate = used_ranges.first.next_sib
987
982
 
@@ -1003,8 +998,55 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1003
998
  candidate.to_s
1004
999
  end
1005
1000
 
1001
+ # Add a new secondary IP range to the given subnet, if it doesn't
1002
+ # already exist
1003
+ def addSecondaryRange(subnet, cidr, name)
1004
+ subnet = getSubnet(cloud_id: subnet, name: subnet, subnet_mu_name: subnet)
1005
+ if !subnet
1006
+ raise MuError, "#{self.to_s} failed to locate a subnet from '#{subnet}'"
1007
+ end
1008
+
1009
+ secondary_ranges = subnet.cloud_desc.secondary_ip_ranges
1010
+ secondary_ranges ||= []
1011
+ secondary_ranges.each { |r|
1012
+ if r.ip_cidr_range == cidr and r.range_name == name
1013
+ return
1014
+ elsif r.ip_cidr_range == cidr or r.range_name == name
1015
+ MU.log "Conflicting secondary IP range, cannot add #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}", MU::WARN, details: r
1016
+ return
1017
+ end
1018
+ }
1019
+
1020
+ secondary_ranges << MU::Cloud::Google.compute(:SubnetworkSecondaryRange).new(
1021
+ ip_cidr_range: cidr,
1022
+ range_name: name
1023
+ )
1024
+ MU.log "Adding new secondary IP range #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}"
1025
+ subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
1026
+ name: subnet.cloud_desc.name,
1027
+ secondary_ip_ranges: secondary_ranges,
1028
+ fingerprint: subnet.cloud_desc.fingerprint
1029
+ )
1030
+ MU::Cloud::Google.compute(credentials: @credentials).patch_subnetwork(@project_id, subnet.az, subnet.cloud_desc.name, subnetobj)
1031
+ end
1032
+
1033
+ def connector(id: nil, name: nil)
1034
+ end
1035
+
1006
1036
  private
1007
1037
 
1038
+ # @return [Array<NetAddr::IPv4Net>]
1039
+ def listSubnetRanges
1040
+ ranges = []
1041
+ subnets.each { |s|
1042
+ ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
1043
+ if s.cloud_desc.secondary_ip_ranges
1044
+ ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
1045
+ end
1046
+ }
1047
+ ranges
1048
+ end
1049
+
1008
1050
  def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
1009
1051
  private_acl = {
1010
1052
  "name" => vpc_name+"-rt",
@@ -1209,6 +1251,10 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1209
1251
  # @return [Boolean]
1210
1252
  def private?
1211
1253
  @parent.cloud_desc
1254
+ if !@parent.routes
1255
+ MU.log "Failed to retrieve routes from #{@parent.to_s}", MU::WARN
1256
+ return true
1257
+ end
1212
1258
  @parent.routes.map { |r|
1213
1259
  if r.dest_range == "0.0.0.0/0" and !r.next_hop_gateway.nil? and
1214
1260
  (r.tags.nil? or r.tags.size == 0) and
@@ -29,8 +29,6 @@ module MU
29
29
  @@authorizers = {}
30
30
  @@acct_to_profile_map = {}
31
31
  @@enable_semaphores = {}
32
- @@readonly_semaphore = Mutex.new
33
- @@readonly = {}
34
32
 
35
33
  # Module used by {MU::Cloud} to insert additional instance methods into
36
34
  # instantiated resources in this cloud layer.
@@ -40,7 +38,7 @@ module MU
40
38
  # @return [String]
41
39
  def url
42
40
  desc = cloud_desc
43
- (desc and desc.self_link) ? desc.self_link : nil
41
+ (desc and desc.respond_to?(:self_link) and desc.self_link) ? desc.self_link : nil
44
42
  end
45
43
  end
46
44
 
@@ -52,6 +50,12 @@ module MU
52
50
  [:url]
53
51
  end
54
52
 
53
+ # Cloud-specific resource methods or attributes we want exposed for
54
+ # reading by {MU::Cloud}
55
+ def self.customAttrReaders
56
+ [:project_id, :customer]
57
+ end
58
+
55
59
  # Is this a "real" cloud provider, or a stub like CloudFormation?
56
60
  def self.virtual?
57
61
  false
@@ -75,12 +79,6 @@ module MU
75
79
  # @param cloudobj [MU::Cloud]
76
80
  # @param deploy [MU::MommaCat]
77
81
  def self.resourceInitHook(cloudobj, deploy)
78
- class << self
79
- attr_reader :project_id
80
- attr_reader :customer
81
- # url is too complex for an attribute (we get it from the cloud API),
82
- # so it's up in AdditionalResourceMethods instead
83
- end
84
82
  return if !cloudobj
85
83
 
86
84
  cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials']))
@@ -96,9 +94,6 @@ module MU
96
94
  cloudobj.instance_variable_set(:@project_id, cloudobj.config['project'])
97
95
  end
98
96
 
99
- # XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe
100
- # that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end
101
-
102
97
  # XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
103
98
  end
104
99
 
@@ -346,8 +341,10 @@ module MU
346
341
  end
347
342
 
348
343
  # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
349
- # @param deploy_id [String]: The deploy for which we're writing the secret
344
+ # @param deploy [String]: The deploy for which we're writing the secret
350
345
  # @param value [String]: The contents of the secret
346
+ # @param name [String]: File/object name
347
+ # @param credentials [String]
351
348
  def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
352
349
  deploy_id = deploy.deploy_id
353
350
  name ||= deploy_id+"-secret"
@@ -847,7 +844,7 @@ MU.log e.message, MU::WARN, details: e.inspect
847
844
  require 'google/apis/iam_v1'
848
845
 
849
846
  if subclass.nil?
850
- @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials)
847
+ @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, retry_readonly: true)
851
848
  return @@iam_api[credentials]
852
849
  elsif subclass.is_a?(Symbol)
853
850
  return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
@@ -863,28 +860,12 @@ MU.log e.message, MU::WARN, details: e.inspect
863
860
  # dopey extra warnings about falling back on scopes
864
861
  credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)
865
862
 
866
- writescopes = ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema']
867
- readscopes = ['admin.directory.group.member.readonly', 'admin.directory.group.readonly', 'admin.directory.user.readonly', 'admin.directory.domain.readonly', 'admin.directory.orgunit.readonly', 'admin.directory.rolemanagement.readonly', 'admin.directory.customer.readonly', 'admin.directory.user.alias.readonly', 'admin.directory.userschema.readonly']
868
- @@readonly_semaphore.synchronize {
869
- use_scopes = readscopes+writescopes
870
- if @@readonly[credentials] and @@readonly[credentials]["AdminDirectoryV1"]
871
- use_scopes = readscopes.dup
872
- end
873
-
874
- if subclass.nil?
875
- begin
876
- @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
877
- rescue Signet::AuthorizationError
878
- MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
879
- @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
880
- @@readonly[credentials] ||= {}
881
- @@readonly[credentials]["AdminDirectoryV1"] = true
882
- end
883
- return @@admin_directory_api[credentials]
884
- elsif subclass.is_a?(Symbol)
885
- return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
886
- end
887
- }
863
+ if subclass.nil?
864
+ @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true, retry_readonly: true)
865
+ return @@admin_directory_api[credentials]
866
+ elsif subclass.is_a?(Symbol)
867
+ return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
868
+ end
888
869
  end
889
870
 
890
871
  # Google's Cloud Resource Manager API
@@ -896,7 +877,7 @@ MU.log e.message, MU::WARN, details: e.inspect
896
877
  if !MU::Cloud::Google.credConfig(credentials)
897
878
  raise MuError, "No such credential set #{credentials} defined in mu.yaml!"
898
879
  end
899
- @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
880
+ @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], retry_readonly: true)
900
881
  return @@resource_api[credentials]
901
882
  elsif subclass.is_a?(Symbol)
902
883
  return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
@@ -1110,7 +1091,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1110
1091
  # Create a Google Cloud Platform API client
1111
1092
  # @param api [String]: Which API are we wrapping?
1112
1093
  # @param scopes [Array<String>]: Google auth scopes applicable to this API
1113
- def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
1094
+ def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false, retry_readonly: false)
1114
1095
  @credentials = credentials
1115
1096
  @scopes = scopes.map { |s|
1116
1097
  if !s.match(/\//) # allow callers to use shorthand
@@ -1127,6 +1108,22 @@ MU.log e.message, MU::WARN, details: e.inspect
1127
1108
  @api.authorization.sub = @masquerade
1128
1109
  @api.authorization.fetch_access_token!
1129
1110
  rescue Signet::AuthorizationError => e
1111
+ if retry_readonly
1112
+ newscopes = @scopes.map { |s|
1113
+ if s =~ /\/cloud-platform$/
1114
+ s += ".read-only"
1115
+ elsif s !~ /\/cloud-platform\b/ and s !~ /\.readonly$/
1116
+ s += ".readonly"
1117
+ end
1118
+ s
1119
+ }
1120
+ if newscopes != @scopes
1121
+ MU.log "Falling back to read-only access to #{api} API on credential set '#{credentials}'", MU::WARN, details: @scopes
1122
+ @scopes = newscopes
1123
+ @api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials)
1124
+ retry
1125
+ end
1126
+ end
1130
1127
  if auth_error_quiet
1131
1128
  MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
1132
1129
  else
@@ -1281,7 +1278,7 @@ MU.log e.message, MU::WARN, details: e.inspect
1281
1278
  raise e
1282
1279
  end
1283
1280
  rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
1284
- if e.message.match(/^quotaExceeded: Request rate/)
1281
+ if e.message.match(/^quotaExceeded: Request rate|failedPrecondition.*?already in progress/)
1285
1282
  if retries <= 10
1286
1283
  sleep wait_backoff
1287
1284
  retries += 1
@@ -1457,6 +1454,9 @@ MU.log e.message, MU::WARN, details: e.inspect
1457
1454
  faked_args.pop
1458
1455
  faked_args.pop
1459
1456
  end
1457
+ if method_sym == :patch_subnetwork
1458
+ faked_args.pop
1459
+ end
1460
1460
  faked_args.push(cloud_id)
1461
1461
  if get_method == :get_project_location_cluster
1462
1462
  faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
data/modules/mu.rb CHANGED
@@ -701,7 +701,7 @@ module MU
701
701
  !$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
702
702
  @@mu_public_addr = $MU_CFG['public_address']
703
703
  if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/) and
704
- File.exists?("/etc/hostname") and File.exists?("/etc/hosts")
704
+ File.exist?("/etc/hostname") and File.exist?("/etc/hosts")
705
705
  hostname = IO.readlines("/etc/hostname")[0].gsub(/\n/, '')
706
706
 
707
707
  hostlines = File.open('/etc/hosts').grep(/.*#{hostname}.*/)
@@ -788,7 +788,7 @@ module MU
788
788
  end
789
789
 
790
790
  # The version of Chef we will install on nodes.
791
- @@chefVersion = "14.0.190"
791
+ @@chefVersion = "18.5.0"
792
792
  # The version of Chef we will install on nodes.
793
793
  # @return [String]
794
794
  def self.chefVersion
@@ -1072,23 +1072,23 @@ module MU
1072
1072
  end
1073
1073
 
1074
1074
 
1075
- # Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
1075
+ # Generate a random password which will satisfy the complexity requirements of things like the stock Amazon Windows AMIs.
1076
1076
  # return [String]: A password string.
1077
- def self.generateWindowsPassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', retries: 50)
1077
+ def self.generatePassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', length: 14, retries: 50)
1078
1078
  # We have dopey complexity requirements, be stringent here.
1079
1079
  # I'll be nice and not condense this into one elegant-but-unreadable regular expression
1080
1080
  attempts = 0
1081
1081
  safe_metachars = Regexp.escape(safe_pattern)
1082
1082
  begin
1083
1083
  if attempts > retries
1084
- MU.log "Failed to generate an adequate Windows password after #{attempts} attempts", MU::ERR
1085
- raise MuError, "Failed to generate an adequate Windows password after #{attempts} attempts"
1084
+ MU.log "Failed to generate an adequate password after #{attempts} attempts", MU::ERR
1085
+ raise MuError, "Failed to generate an adequate password after #{attempts} attempts"
1086
1086
  end
1087
- winpass = Password.random(14..16)
1087
+ winpass = Password.random(length..(length+2))
1088
1088
  attempts += 1
1089
1089
  end while winpass.nil? or !winpass.match(/^[a-z]/i) or !winpass.match(/[A-Z]/) or !winpass.match(/[a-z]/) or !winpass.match(/\d/) or !winpass.match(/[#{safe_metachars}]/) or winpass.match(/[^\w\d#{safe_metachars}]/)
1090
1090
 
1091
- MU.log "Generated Windows password after #{attempts} attempts", MU::DEBUG
1091
+ MU.log "Generated password after #{attempts} attempts", MU::DEBUG
1092
1092
  return winpass
1093
1093
  end
1094
1094
 
@@ -0,0 +1,46 @@
1
+ # groomers: Ansible
2
+ ---
3
+ appname: smoketest
4
+ generate_passwords:
5
+ - itemname: elasticpw
6
+ minlength: 12
7
+ vpcs:
8
+ - name: wrapper
9
+ servers:
10
+ - name: frontend
11
+ platform: centos7
12
+ groomer: Ansible
13
+ vpc:
14
+ name: wrapper
15
+ size: t3.medium
16
+ vault_access:
17
+ - item: elasticpw
18
+ run_list:
19
+ - mu-logstash
20
+ #<% if cloud != "AWS" %>
21
+ - name: backend
22
+ platform: centos7
23
+ groomer: Ansible
24
+ vpc:
25
+ name: wrapper
26
+ size: m5.large
27
+ vault_access:
28
+ - item: elasticpw
29
+ run_list:
30
+ - mu-elastic
31
+ #<% else %>
32
+ #search_domains:
33
+ #- name: logsearch
34
+ # elasticsearch_version: '7.4'
35
+ # instance_count: 1
36
+ # instance_type: r5.large.elasticsearch
37
+ # ebs_size: 10
38
+ # ebs_type: gp2
39
+ # access_policies:
40
+ # Version: '2012-10-17'
41
+ # Statement:
42
+ # - Effect: Allow
43
+ # Principal:
44
+ # AWS: "*"
45
+ # Action: es:ESHttp*
46
+ #<% end %>
@@ -15,7 +15,7 @@ control 'init' do
15
15
  node = json('/tmp/chef_node.json').params
16
16
  NODE_PUB_IP=node_meta[0]['pub_ip']
17
17
  CHEF_SERVER_VERSION="12.17.15-1"
18
- CHEF_CLIENT_VERSION="14.0.190"
18
+ CHEF_CLIENT_VERSION="18.2.7"
19
19
  KNIFE_WINDOWS="1.9.0"
20
20
  MU_BASE="/opt/mu"
21
21
  f = "/etc/ssh/sshd_config"