cloud-mu 3.5.0 → 3.6.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +5 -2
  3. data/Berksfile.lock +135 -0
  4. data/ansible/roles/mu-base/README.md +33 -0
  5. data/ansible/roles/mu-base/defaults/main.yml +2 -0
  6. data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
  7. data/ansible/roles/mu-base/files/check_apm.sh +18 -0
  8. data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
  9. data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
  10. data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
  11. data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
  12. data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
  13. data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
  14. data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
  15. data/ansible/roles/mu-base/files/logrotate.conf +35 -0
  16. data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
  17. data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
  18. data/ansible/roles/mu-base/handlers/main.yml +5 -0
  19. data/ansible/roles/mu-base/meta/main.yml +53 -0
  20. data/ansible/roles/mu-base/tasks/main.yml +113 -0
  21. data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
  22. data/ansible/roles/mu-base/tests/inventory +2 -0
  23. data/ansible/roles/mu-base/tests/test.yml +5 -0
  24. data/ansible/roles/mu-base/vars/main.yml +1 -0
  25. data/ansible/roles/mu-compliance/README.md +33 -0
  26. data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
  27. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
  28. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
  29. data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
  30. data/ansible/roles/mu-compliance/meta/main.yml +53 -0
  31. data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
  32. data/ansible/roles/mu-compliance/tests/inventory +2 -0
  33. data/ansible/roles/mu-compliance/tests/test.yml +5 -0
  34. data/ansible/roles/mu-compliance/vars/main.yml +4 -0
  35. data/ansible/roles/mu-elastic/README.md +51 -0
  36. data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
  37. data/ansible/roles/mu-elastic/files/jvm.options +93 -0
  38. data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
  39. data/ansible/roles/mu-elastic/meta/main.yml +52 -0
  40. data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
  41. data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
  42. data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
  43. data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
  44. data/ansible/roles/mu-elastic/tests/inventory +2 -0
  45. data/ansible/roles/mu-elastic/tests/test.yml +5 -0
  46. data/ansible/roles/mu-elastic/vars/main.yml +2 -0
  47. data/ansible/roles/mu-logstash/README.md +51 -0
  48. data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
  49. data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
  50. data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
  51. data/ansible/roles/mu-logstash/files/jvm.options +84 -0
  52. data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
  53. data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
  54. data/ansible/roles/mu-logstash/meta/main.yml +52 -0
  55. data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
  56. data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
  57. data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
  58. data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
  59. data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
  60. data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
  61. data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
  62. data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
  63. data/ansible/roles/mu-logstash/tests/inventory +2 -0
  64. data/ansible/roles/mu-logstash/tests/test.yml +5 -0
  65. data/ansible/roles/mu-logstash/vars/main.yml +2 -0
  66. data/ansible/roles/mu-rdp/README.md +33 -0
  67. data/ansible/roles/mu-rdp/meta/main.yml +53 -0
  68. data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
  69. data/ansible/roles/mu-rdp/tests/inventory +2 -0
  70. data/ansible/roles/mu-rdp/tests/test.yml +5 -0
  71. data/ansible/roles/mu-windows/tasks/main.yml +3 -0
  72. data/bin/mu-ansible-secret +1 -1
  73. data/bin/mu-aws-setup +4 -3
  74. data/bin/mu-azure-setup +5 -5
  75. data/bin/mu-configure +25 -17
  76. data/bin/mu-firewall-allow-clients +1 -0
  77. data/bin/mu-gcp-setup +3 -3
  78. data/bin/mu-load-config.rb +1 -0
  79. data/bin/mu-node-manage +66 -33
  80. data/bin/mu-self-update +2 -2
  81. data/bin/mu-upload-chef-artifacts +6 -1
  82. data/bin/mu-user-manage +1 -1
  83. data/cloud-mu.gemspec +25 -23
  84. data/cookbooks/firewall/CHANGELOG.md +417 -224
  85. data/cookbooks/firewall/LICENSE +202 -0
  86. data/cookbooks/firewall/README.md +153 -126
  87. data/cookbooks/firewall/TODO.md +6 -0
  88. data/cookbooks/firewall/attributes/firewalld.rb +7 -0
  89. data/cookbooks/firewall/attributes/iptables.rb +3 -3
  90. data/cookbooks/firewall/chefignore +115 -0
  91. data/cookbooks/firewall/libraries/helpers.rb +5 -0
  92. data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
  93. data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
  94. data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
  95. data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
  96. data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
  97. data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
  98. data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
  99. data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
  100. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
  101. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
  102. data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
  103. data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
  104. data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
  105. data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
  106. data/cookbooks/firewall/metadata.json +40 -1
  107. data/cookbooks/firewall/metadata.rb +15 -0
  108. data/cookbooks/firewall/recipes/default.rb +7 -7
  109. data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
  110. data/cookbooks/firewall/recipes/firewalld.rb +87 -0
  111. data/cookbooks/firewall/renovate.json +18 -0
  112. data/cookbooks/firewall/resources/firewalld.rb +28 -0
  113. data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
  114. data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
  115. data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
  116. data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
  117. data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
  118. data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
  119. data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
  120. data/cookbooks/firewall/resources/nftables.rb +71 -0
  121. data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
  122. data/cookbooks/mu-activedirectory/Berksfile +1 -1
  123. data/cookbooks/mu-activedirectory/metadata.rb +1 -1
  124. data/cookbooks/mu-firewall/metadata.rb +2 -2
  125. data/cookbooks/mu-master/Berksfile +4 -3
  126. data/cookbooks/mu-master/attributes/default.rb +5 -2
  127. data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
  128. data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
  129. data/cookbooks/mu-master/libraries/mu.rb +24 -0
  130. data/cookbooks/mu-master/metadata.rb +5 -5
  131. data/cookbooks/mu-master/recipes/default.rb +31 -20
  132. data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
  133. data/cookbooks/mu-master/recipes/init.rb +58 -19
  134. data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
  135. data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
  136. data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
  137. data/cookbooks/mu-php54/Berksfile +1 -1
  138. data/cookbooks/mu-php54/metadata.rb +2 -2
  139. data/cookbooks/mu-tools/Berksfile +2 -3
  140. data/cookbooks/mu-tools/attributes/default.rb +3 -4
  141. data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
  142. data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
  143. data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
  144. data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
  145. data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
  146. data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
  147. data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
  148. data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
  149. data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
  150. data/cookbooks/mu-tools/libraries/helper.rb +21 -9
  151. data/cookbooks/mu-tools/metadata.rb +4 -4
  152. data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
  153. data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
  154. data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
  155. data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
  156. data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
  157. data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
  158. data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
  159. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  160. data/data_bags/nagios_services/apm_backend_connect.json +5 -0
  161. data/data_bags/nagios_services/apm_listen.json +5 -0
  162. data/data_bags/nagios_services/elastic_shards.json +5 -0
  163. data/data_bags/nagios_services/logstash.json +5 -0
  164. data/data_bags/nagios_services/rhel7_updates.json +8 -0
  165. data/extras/image-generators/AWS/centos7.yaml +1 -0
  166. data/extras/image-generators/AWS/rhel7.yaml +21 -0
  167. data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
  168. data/extras/image-generators/AWS/win2k16.yaml +1 -0
  169. data/extras/image-generators/AWS/win2k19.yaml +1 -0
  170. data/extras/list-stock-amis +0 -0
  171. data/extras/ruby_rpm/muby.spec +8 -5
  172. data/extras/vault_tools/export_vaults.sh +1 -1
  173. data/extras/vault_tools/recreate_vaults.sh +0 -0
  174. data/extras/vault_tools/test_vaults.sh +0 -0
  175. data/install/deprecated-bash-library.sh +1 -1
  176. data/install/installer +4 -2
  177. data/modules/mommacat.ru +3 -1
  178. data/modules/mu/adoption.rb +1 -1
  179. data/modules/mu/cloud/dnszone.rb +2 -2
  180. data/modules/mu/cloud/machine_images.rb +26 -25
  181. data/modules/mu/cloud/resource_base.rb +213 -182
  182. data/modules/mu/cloud/server_pool.rb +1 -1
  183. data/modules/mu/cloud/ssh_sessions.rb +7 -5
  184. data/modules/mu/cloud/wrappers.rb +2 -2
  185. data/modules/mu/cloud.rb +1 -1
  186. data/modules/mu/config/bucket.rb +1 -1
  187. data/modules/mu/config/function.rb +6 -1
  188. data/modules/mu/config/loadbalancer.rb +24 -2
  189. data/modules/mu/config/ref.rb +12 -0
  190. data/modules/mu/config/role.rb +1 -1
  191. data/modules/mu/config/schema_helpers.rb +42 -9
  192. data/modules/mu/config/server.rb +43 -27
  193. data/modules/mu/config/tail.rb +19 -10
  194. data/modules/mu/config.rb +6 -5
  195. data/modules/mu/defaults/AWS.yaml +78 -114
  196. data/modules/mu/deploy.rb +9 -2
  197. data/modules/mu/groomer.rb +12 -4
  198. data/modules/mu/groomers/ansible.rb +104 -20
  199. data/modules/mu/groomers/chef.rb +15 -6
  200. data/modules/mu/master.rb +9 -4
  201. data/modules/mu/mommacat/daemon.rb +4 -2
  202. data/modules/mu/mommacat/naming.rb +1 -2
  203. data/modules/mu/mommacat/storage.rb +7 -2
  204. data/modules/mu/mommacat.rb +33 -6
  205. data/modules/mu/providers/aws/database.rb +161 -8
  206. data/modules/mu/providers/aws/dnszone.rb +11 -6
  207. data/modules/mu/providers/aws/endpoint.rb +81 -6
  208. data/modules/mu/providers/aws/firewall_rule.rb +254 -172
  209. data/modules/mu/providers/aws/function.rb +65 -3
  210. data/modules/mu/providers/aws/loadbalancer.rb +39 -28
  211. data/modules/mu/providers/aws/log.rb +2 -1
  212. data/modules/mu/providers/aws/role.rb +25 -7
  213. data/modules/mu/providers/aws/server.rb +36 -12
  214. data/modules/mu/providers/aws/server_pool.rb +237 -127
  215. data/modules/mu/providers/aws/storage_pool.rb +7 -1
  216. data/modules/mu/providers/aws/user.rb +1 -1
  217. data/modules/mu/providers/aws/userdata/linux.erb +6 -2
  218. data/modules/mu/providers/aws/userdata/windows.erb +7 -5
  219. data/modules/mu/providers/aws/vpc.rb +49 -25
  220. data/modules/mu/providers/aws.rb +13 -8
  221. data/modules/mu/providers/azure/container_cluster.rb +1 -1
  222. data/modules/mu/providers/azure/loadbalancer.rb +2 -2
  223. data/modules/mu/providers/azure/server.rb +5 -2
  224. data/modules/mu/providers/azure/userdata/linux.erb +1 -1
  225. data/modules/mu/providers/azure.rb +11 -8
  226. data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
  227. data/modules/mu/providers/google/container_cluster.rb +15 -2
  228. data/modules/mu/providers/google/folder.rb +2 -1
  229. data/modules/mu/providers/google/function.rb +130 -4
  230. data/modules/mu/providers/google/habitat.rb +2 -1
  231. data/modules/mu/providers/google/loadbalancer.rb +407 -160
  232. data/modules/mu/providers/google/role.rb +16 -3
  233. data/modules/mu/providers/google/server.rb +5 -1
  234. data/modules/mu/providers/google/user.rb +25 -18
  235. data/modules/mu/providers/google/userdata/linux.erb +1 -1
  236. data/modules/mu/providers/google/vpc.rb +53 -7
  237. data/modules/mu/providers/google.rb +39 -39
  238. data/modules/mu.rb +8 -8
  239. data/modules/tests/elk.yaml +46 -0
  240. data/test/mu-master-test/controls/all_in_one.rb +1 -1
  241. metadata +207 -112
  242. data/cookbooks/firewall/CONTRIBUTING.md +0 -2
  243. data/cookbooks/firewall/MAINTAINERS.md +0 -19
  244. data/cookbooks/firewall/libraries/matchers.rb +0 -30
  245. data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -114,20 +114,6 @@ module MU
114
114
  raise MuError, "VPC endpoint failed #{endpoint_id}: #{resp}" if resp.state == "failed"
115
115
  end
116
116
 
117
- if @config["enable_traffic_logging"]
118
- loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
119
- logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
120
-
121
- MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup.mu_name}"
122
- MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
123
- resource_ids: [@cloud_id],
124
- resource_type: "VPC",
125
- traffic_type: "ALL",
126
- log_group_name: loggroup.mu_name,
127
- deliver_logs_permission_arn: logrole.cloudobj.arn
128
- )
129
- end
130
-
131
117
  nat_gateways = create_subnets
132
118
 
133
119
  notify
@@ -270,6 +256,39 @@ module MU
270
256
  }
271
257
  end
272
258
 
259
+ if @config["enable_traffic_logging"]
260
+ ext = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_flow_logs(
261
+ filter: [
262
+ { name: "resource-id", values: [@cloud_id] }
263
+ ]
264
+ )
265
+ # XXX a smarter guard would filter with more specificity
266
+ if !ext or ext.flow_logs.empty?
267
+ loggroup = if @config['log_group_name']
268
+ @config['log_group_name']
269
+ else
270
+ @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs").mu_name
271
+ end
272
+ logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
273
+
274
+
275
+ MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup}"
276
+ MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
277
+ resource_ids: [@cloud_id],
278
+ resource_type: "VPC",
279
+ traffic_type: "ALL",
280
+ log_group_name: loggroup,
281
+ deliver_logs_permission_arn: logrole.cloudobj.arn,
282
+ tag_specifications: [
283
+ {
284
+ resource_type: "vpc-flow-log",
285
+ tags: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
286
+ }
287
+ ]
288
+ )
289
+ end
290
+ end
291
+
273
292
  end
274
293
 
275
294
  # Locate an existing VPC or VPCs and return an array containing matching AWS resource descriptors for those that match.
@@ -942,13 +961,15 @@ module MU
942
961
  ok = true
943
962
 
944
963
  if vpc["enable_traffic_logging"]
945
- logdesc = {
946
- "name" => vpc['name']+"loggroup",
947
- }
948
- logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
949
- # logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
950
- configurator.insertKitten(logdesc, "logs")
951
- MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
964
+ if !vpc['log_group_name']
965
+ logdesc = {
966
+ "name" => vpc['name']+"loggroup",
967
+ }
968
+ logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
969
+ # logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
970
+ configurator.insertKitten(logdesc, "logs")
971
+ MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
972
+ end
952
973
 
953
974
  roledesc = {
954
975
  "name" => vpc['name']+"logrole",
@@ -971,20 +992,23 @@ module MU
971
992
  "targets" => [
972
993
  {
973
994
  "type" => "log",
974
- "identifier" => vpc['name']+"loggroup"
995
+ "identifier" => vpc['log_group_name'] ? vpc['log_group_name'] : vpc['name']+"loggroup"
975
996
  }
976
997
  ]
977
998
  }
978
- ],
979
- "dependencies" => [
999
+ ]
1000
+ }
1001
+ if !vpc['log_group_name']
1002
+ roledesc["dependencies"] = [
980
1003
  {
981
1004
  "type" => "log",
982
1005
  "name" => vpc['name']+"loggroup"
983
1006
  }
984
1007
  ]
985
- }
1008
+ end
986
1009
  roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
987
1010
  roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
1011
+
988
1012
  configurator.insertKitten(roledesc, "roles")
989
1013
  MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
990
1014
  end
@@ -44,6 +44,12 @@ module MU
44
44
  [cfg['account_number']]
45
45
  end
46
46
 
47
+ # Cloud-specific resource methods or attributes we want exposed for
48
+ # reading by {MU::Cloud}
49
+ def self.customAttrReaders
50
+ [:region, :cloudformation_data]
51
+ end
52
+
47
53
  # A hook that is always called just before any of the instance method of
48
54
  # our resource implementations gets invoked, so that we can ensure that
49
55
  # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -51,10 +57,6 @@ module MU
51
57
  # @param cloudobj [MU::Cloud]
52
58
  # @param _deploy [MU::MommaCat]
53
59
  def self.resourceInitHook(cloudobj, _deploy)
54
- class << self
55
- attr_reader :cloudformation_data
56
- attr_reader :region
57
- end
58
60
  return if !cloudobj
59
61
  cloudobj.instance_variable_set(:@cloudformation_data, {})
60
62
 
@@ -409,8 +411,10 @@ end
409
411
  end
410
412
 
411
413
  # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
412
- # @param deploy_id [String]: The deploy for which we're writing the secret
414
+ # @param deploy [String]: The deploy for which we're writing the secret
413
415
  # @param value [String]: The contents of the secret
416
+ # @param name [String]: File/object name
417
+ # @param credentials [String]
414
418
  def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
415
419
  require "aws-sdk-s3"
416
420
  name ||= deploy.deploy_id+"-secret"
@@ -531,9 +535,9 @@ end
531
535
  if !@@is_in_aws.nil?
532
536
  return @@is_in_aws
533
537
  end
534
-
538
+ start = Time.now
535
539
  begin
536
- Timeout.timeout(4) do
540
+ Timeout.timeout(10) do
537
541
  instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
538
542
  if !instance_id.nil? and instance_id.size > 0
539
543
  @@is_in_aws = true
@@ -550,6 +554,7 @@ end
550
554
  end
551
555
  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH
552
556
  end
557
+ MU.log "Fetch of http://169.254.169.254/latest/meta-data/instance-id took #{sprintf("%.2fs", Time.now-start)}", MU::WARN
553
558
 
554
559
  @@is_in_aws = false
555
560
  false
@@ -978,7 +983,7 @@ end
978
983
  id = matches.first
979
984
  elsif matches.size == 0
980
985
  if raise_on_missing
981
- raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
986
+ raise MuError, "No IAM or ACM certificate named #{name} was found in #{region} (credentials: #{(credentials.nil? or credentials.empty?) ? "default" : credentials})"
982
987
  else
983
988
  return nil
984
989
  end
@@ -249,7 +249,7 @@ module MU
249
249
  os_obj = MU::Cloud::Azure.containers(:ContainerServiceWindowsProfile, model_version: "V2019_02_01").new
250
250
  os_obj.admin_username = "muadmin"
251
251
  # Azure password constraints are extra-annoying
252
- winpass = MU.generateWindowsPassword(safe_pattern: '!@#$%^&*()', retries: 150)
252
+ winpass = MU.generatePassword(safe_pattern: '!@#$%^&*()', retries: 150)
253
253
  # TODO store this somewhere the user can get at it
254
254
  os_obj.admin_password = winpass
255
255
  os_obj
@@ -50,8 +50,8 @@ module MU
50
50
  # Register a Server node with an existing LoadBalancer.
51
51
  #
52
52
  # @param instance_id [String] A node to register.
53
- # @param targetgroups [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
54
- def registerNode(instance_id, targetgroups: nil)
53
+ # @param backends [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
54
+ def registerTarget(instance_id, backends: nil)
55
55
  end
56
56
 
57
57
  # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -426,7 +426,10 @@ MU.log "Azure::Server.find called", MU::NOTICE, details: args
426
426
  # @param size [String]: Size (in gb) of the new volume
427
427
  # @param type [String]: Cloud storage type of the volume, if applicable
428
428
  # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
429
- def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
429
+ def addVolume(dev: nil, size: 0, type: "pd-standard", delete_on_termination: false)
430
+ if dev.nil? or size == 0
431
+ raise MuError, "Must specify a device name and a size for addVolume"
432
+ end
430
433
  end
431
434
 
432
435
  # Determine whether the node in question exists at the Cloud provider
@@ -785,7 +788,7 @@ MU.log "Azure::Server.find called", MU::NOTICE, details: args
785
788
  os_obj.admin_password = begin
786
789
  @deploy.fetchSecret(@mu_name, "windows_admin_password")
787
790
  rescue MU::MommaCat::SecretError
788
- pw = MU.generateWindowsPassword
791
+ pw = MU.generatePassword
789
792
  @deploy.saveNodeSecret(@mu_name, pw, "windows_admin_password")
790
793
  pw
791
794
  end
@@ -105,7 +105,7 @@ umask 0077
105
105
 
106
106
  # Install Chef now, because why not?
107
107
  if [ ! -f /opt/chef/embedded/bin/ruby ];then
108
- curl https://www.chef.io/chef/install.sh > chef-install.sh
108
+ curl https://omnitruck.chef.io/install.sh > chef-install.sh
109
109
  set +e
110
110
  # We may run afoul of a synchronous bootstrap process doing the same thing. So
111
111
  # wait until we've managed to run successfully.
@@ -52,6 +52,12 @@ module MU
52
52
  []
53
53
  end
54
54
 
55
+ # Cloud-specific resource methods or attributes we want exposed for
56
+ # reading by {MU::Cloud}
57
+ def self.customAttrReaders
58
+ [:region, :resource_group]
59
+ end
60
+
55
61
  # A hook that is always called just before any of the instance method of
56
62
  # our resource implementations gets invoked, so that we can ensure that
57
63
  # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -59,9 +65,6 @@ module MU
59
65
  # @param cloudobj [MU::Cloud]
60
66
  # @param deploy [MU::MommaCat]
61
67
  def self.resourceInitHook(cloudobj, deploy)
62
- class << self
63
- attr_reader :resource_group
64
- end
65
68
  return if !cloudobj
66
69
 
67
70
  rg = if !deploy
@@ -565,12 +568,12 @@ MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
565
568
  return @@metadata if svc == "instance" and @@metadata
566
569
  base_url = "http://169.254.169.254/metadata/#{svc}"
567
570
  args["api-version"] = api_version
568
- arg_str = args.keys.map { |k| k.to_s+"="+args[k].to_s }.join("&")
571
+ arg_str = args.keys.sort.map { |k| k.to_s+"="+CGI.escape(args[k].to_s) }.join("&")
569
572
 
570
573
  begin
571
574
  Timeout.timeout(2) do
572
- resp = JSON.parse(URI.open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
573
- MU.log "curl -H Metadata:true "+"#{base_url}/?#{arg_str}", loglevel, details: resp
575
+ resp = JSON.parse(URI.open("#{base_url}?#{arg_str}","Metadata"=>"true").read)
576
+ MU.log "curl -H Metadata:true "+"#{base_url}?#{arg_str}", loglevel, details: resp
574
577
  if svc != "instance"
575
578
  return resp
576
579
  else
@@ -597,9 +600,9 @@ MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
597
600
  cfg = credConfig(credentials)
598
601
 
599
602
  if cfg and MU::Cloud::Azure.hosted?
600
- token = MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2018-02-01", args: { "resource"=>"https://management.azure.com/" })
603
+ token = MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2020-09-01", args: { "resource"=>"https://management.azure.com/" })
601
604
  if !token
602
- MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2018-02-01", args: { "resource"=>"https://management.azure.com/" }, debug: true)
605
+ MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2020-09-01", args: { "resource"=>"https://management.azure.com/" }, debug: true)
603
606
  raise MuError, "Failed to get machine oauth token"
604
607
  end
605
608
  machine = MU::Cloud::Azure.get_metadata
@@ -248,7 +248,7 @@ module MU
248
248
  end
249
249
  # Placeholder. This is a NOOP for CloudFormation, which doesn't build
250
250
  # resources directly.
251
- def self.genericMuDNSEntry(*args)
251
+ def self.genericMuDNSEntry(**args)
252
252
  MU.log "find() not implemented for CloudFormation layer", MU::DEBUG
253
253
  nil
254
254
  end
@@ -95,7 +95,7 @@ module MU
95
95
  desc = {
96
96
  :name => @mu_name.downcase,
97
97
  :description => @deploy.deploy_id,
98
- :network => @vpc.cloud_id,
98
+ :network => @vpc.url,
99
99
  :enable_tpu => @config['tpu'],
100
100
  :resource_labels => labels,
101
101
  :locations => locations,
@@ -108,6 +108,7 @@ module MU
108
108
  ),
109
109
  }
110
110
 
111
+
111
112
  if @config['kubernetes']
112
113
  desc[:addons_config] = MU::Cloud::Google.container(:AddonsConfig).new(
113
114
  horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
@@ -135,6 +136,8 @@ module MU
135
136
  end
136
137
  }
137
138
  end
139
+
140
+
138
141
  if @config['log_facility'] == "kubernetes"
139
142
  desc[:logging_service] = "logging.googleapis.com/kubernetes"
140
143
  desc[:monitoring_service] = "monitoring.googleapis.com/kubernetes"
@@ -183,6 +186,7 @@ module MU
183
186
  )
184
187
  end
185
188
 
189
+
186
190
  if @config['ip_aliases'] or @config['custom_subnet'] or
187
191
  @config['services_ip_block'] or @config['services_ip_block_name'] or
188
192
  @config['pod_ip_block'] or @config['pod_ip_block_name'] or
@@ -210,17 +214,26 @@ module MU
210
214
  end
211
215
 
212
216
  if @config['services_ip_block']
217
+ if @vpc.project_id != @project_id
218
+ alloc_desc[:services_secondary_range_name] ||= @config['name']+"-services"
219
+ @vpc.addSecondaryRange(desc[:subnetwork], @config['services_ip_block'], alloc_desc[:services_secondary_range_name])
220
+
221
+ end
213
222
  alloc_desc[:services_ipv4_cidr_block] = @config['services_ip_block']
214
223
  end
215
224
  if @config['tpu_ip_block']
216
225
  alloc_desc[:tpu_ipv4_cidr_block] = @config['tpu_ip_block']
217
226
  end
218
227
  if @config['pod_ip_block']
228
+ if @vpc.project_id != @project_id
229
+ alloc_desc[:cluster_secondary_range_name] ||= @config['name']+"-pods"
230
+ @vpc.addSecondaryRange(desc[:subnetwork], @config['pod_ip_block'], alloc_desc[:cluster_secondary_range_name])
231
+
232
+ end
219
233
  alloc_desc[:cluster_ipv4_cidr_block] = @config['pod_ip_block']
220
234
  end
221
235
 
222
236
  desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(alloc_desc)
223
- pp alloc_desc
224
237
  end
225
238
 
226
239
  if @config['authorized_networks'] and @config['authorized_networks'].size > 0
@@ -260,7 +260,8 @@ module MU
260
260
  args[:flags]['parent_id']
261
261
  else
262
262
  my_org = MU::Cloud::Google.getOrg(args[:credentials])
263
- my_org.name
263
+ # XXX re-raise with a clear permission error
264
+ my_org.name if my_org
264
265
  end
265
266
 
266
267
  if args[:cloud_id]
@@ -120,7 +120,7 @@ module example.com/cloudfunction
120
120
  def groom
121
121
  desc = {}
122
122
 
123
- func_obj = buildDesc
123
+ func_obj = buildDesc(true)
124
124
 
125
125
  labels = Hash[@tags.keys.map { |k|
126
126
  [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
@@ -161,6 +161,13 @@ module example.com/cloudfunction
161
161
  cloud_desc.event_trigger.resource != @config['triggers'].first['resource']
162
162
  need_update = true
163
163
  end
164
+ elsif !cloud_desc.https_trigger
165
+ need_update = true
166
+ end
167
+
168
+ if (@config['internal_only'] and cloud_desc.ingress_settings != "ALLOW_INTERNAL_ONLY") or
169
+ (!@config['internal_only'] and cloud_desc.ingress_settings != "ALLOW_ALL")
170
+ need_update = true
164
171
  end
165
172
 
166
173
  current = Dir.mktmpdir(@mu_name+"-current") { |dir|
@@ -206,6 +213,7 @@ module example.com/cloudfunction
206
213
  end
207
214
 
208
215
  if need_update
216
+ MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
209
217
  MU.log "Updating Cloud Function #{@cloud_id}", MU::NOTICE, details: func_obj
210
218
  begin
211
219
  MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
@@ -235,6 +243,71 @@ module example.com/cloudfunction
235
243
  tempfile.unlink
236
244
  end
237
245
 
246
+ policy = MU::Cloud::Google.function(credentials: @credentials).get_project_location_function_iam_policy(@cloud_id)
247
+
248
+ if @config['allow_unauthenticated'] and !allowsUnauthencated?
249
+ policy ||= MU::Cloud::Google.function(:Policy).new(
250
+ bindings: []
251
+ )
252
+ policy.bindings ||= []
253
+ policy.bindings << MU::Cloud::Google.function(:Binding).new(
254
+ members: ["allUsers"],
255
+ role: "roles/cloudfunctions.invoker"
256
+ )
257
+
258
+ pol_req = MU::Cloud::Google.function(:SetIamPolicyRequest).new(
259
+ policy: policy
260
+ )
261
+ MU.log "Enabling anonymous invocation of Cloud Function #{@mu_name}", MU::NOTICE
262
+ MU::Cloud::Google.function(credentials: @credentials).set_function_iam_policy(@cloud_id, pol_req)
263
+ elsif !@config['allow_unauthenticated'] and allowsUnauthencated?
264
+ policy.bindings.reject! { |b|
265
+ b.members.include?("allUsers") and b.role == "roles/cloudfunctions.invoker"
266
+ }
267
+ pol_req = MU::Cloud::Google.function(:SetIamPolicyRequest).new(
268
+ policy: policy
269
+ )
270
+ MU.log "Disabling anonymous invocation of Cloud Function #{@mu_name}", MU::NOTICE
271
+ MU::Cloud::Google.function(credentials: @credentials).set_function_iam_policy(@cloud_id, pol_req)
272
+ end
273
+
274
+ # If we have a loadbalancer configured, attach us to it
275
+ if !@config['loadbalancers'].nil?
276
+ if @loadbalancers.nil?
277
+ raise MuError, "#{@mu_name} is configured to use LoadBalancers, but none have been loaded by dependencies()"
278
+ end
279
+
280
+ neg_name = @deploy.getResourceName(@config["name"], max_length: 19, never_gen_unique: true).downcase
281
+ neg_desc = begin
282
+ MU::Cloud::Google.compute(credentials: @config['credentials']).get_region_network_endpoint_group(@project_id, @config['region'], neg_name)
283
+ rescue ::Google::Apis::ClientError => e
284
+ raise e if e.message !~ /notFound:/
285
+ neg_obj = MU::Cloud::Google.compute(:NetworkEndpointGroup).new(
286
+ name: neg_name,
287
+ description: @deploy.deploy_id,
288
+ cloud_function: MU::Cloud::Google.compute(:NetworkEndpointGroupCloudFunction).new(
289
+ function: @cloud_id.gsub(/.*?\//, '')
290
+ ),
291
+ network_endpoint_type: "SERVERLESS"
292
+ )
293
+ MU.log "Creating Network Endpoint Group #{neg_name}", details: neg_obj
294
+ MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_network_endpoint_group(@project_id, @config['region'], neg_obj)
295
+ retry
296
+ end
297
+
298
+ @loadbalancers.each { |lb|
299
+ # if !lb.targetgroups
300
+ # MU.retrier([], max: 6, wait: 15, loop_if: Proc.new { !lb.targetgroups }) {
301
+ # lb.cloud_desc(use_cache: false)
302
+ # }
303
+ # end
304
+ # lb.targetgroups.each_pair { |tg_name, tg|
305
+ # addTrigger(tg.target_group_arn, "elasticloadbalancing", tg_name)
306
+ # }
307
+ lb.registerTarget(neg_desc.self_link)
308
+ }
309
+ end
310
+
238
311
  end
239
312
 
240
313
  # Return the metadata for this project's configuration
@@ -330,6 +403,12 @@ module example.com/cloudfunction
330
403
  bok["handler"] = cloud_desc.entry_point
331
404
  bok["timeout"] = cloud_desc.timeout.gsub(/[^\d]/, '').to_i
332
405
 
406
+ if cloud_desc.ingress_settings and cloud_desc.ingress_settings == "ALLOW_INTERNAL_ONLY"
407
+ bok['internal_only'] = true
408
+ else
409
+ bok['internal_only'] = false
410
+ end
411
+
333
412
  if cloud_desc.vpc_connector
334
413
  bok["vpc_connector"] = cloud_desc.vpc_connector
335
414
  elsif cloud_desc.network
@@ -350,7 +429,7 @@ module example.com/cloudfunction
350
429
  type: "vpcs"
351
430
  )
352
431
  end
353
-
432
+
354
433
  if cloud_desc.environment_variables and cloud_desc.environment_variables.size > 0
355
434
  bok['environment_variable'] = cloud_desc.environment_variables.keys.map { |k| { "key" => k, "value" => cloud_desc.environment_variables[k] } }
356
435
  end
@@ -373,6 +452,9 @@ module example.com/cloudfunction
373
452
  'zip_file' => codefile
374
453
  }
375
454
 
455
+
456
+ bok['allow_unauthenticated'] = allowsUnauthencated?
457
+
376
458
  bok
377
459
  end
378
460
 
@@ -411,6 +493,16 @@ module example.com/cloudfunction
411
493
  "type" => "string",
412
494
  "description" => "+DEPRECATED+ VPC Connector to attach, of the form +projects/my-project/locations/some-region/connectors/my-connector+. This option will be removed once proper google-cloud-sdk support for VPC Connectors becomes available, at which point we will piggyback on the normal +vpc+ stanza and resolve connectors as needed."
413
495
  },
496
+ "allow_unauthenticated" => {
497
+ "type" => "boolean",
498
+ "default" => false,
499
+ "description" => "Only applicable for HTTPS-triggered functions; allows function invocation without credentials"
500
+ },
501
+ "internal_only" => {
502
+ "type" => "boolean",
503
+ "default" => false,
504
+ "description" => "Permit only traffic from VPC networks in the same project or VPC SC perimeter"
505
+ },
414
506
  "vpc_connector_allow_all_egress" => {
415
507
  "type" => "boolean",
416
508
  "default" => false,
@@ -473,11 +565,13 @@ module example.com/cloudfunction
473
565
  # @return [String]: The Cloud Storage URL to the result
474
566
  def self.uploadPackage(zipfile, filename, credentials: nil)
475
567
  bucket = MU::Cloud::Google.adminBucketName(credentials)
568
+
476
569
  obj_obj = MU::Cloud::Google.storage(:Object).new(
477
570
  content_type: "application/zip",
478
571
  name: filename
479
572
  )
480
573
 
574
+ MU.log "Uploading #{zipfile} to #{bucket}/#{filename}"
481
575
  MU::Cloud::Google.storage(credentials: credentials).insert_object(
482
576
  bucket,
483
577
  obj_obj,
@@ -574,12 +668,33 @@ module example.com/cloudfunction
574
668
  # ok = false
575
669
  # end
576
670
 
671
+ if !function["loadbalancers"].nil?
672
+ function["loadbalancers"].each { |lb|
673
+ lb["name"] ||= lb["concurrent_load_balancer"]
674
+ if lb["name"]
675
+ MU::Config.addDependency(function, lb["name"], "loadbalancer")
676
+ end
677
+ }
678
+ end
679
+
577
680
  ok
578
681
  end
579
682
 
580
683
  private
581
684
 
582
- def buildDesc
685
+ def allowsUnauthencated?
686
+ policy = MU::Cloud::Google.function(credentials: @credentials).get_project_location_function_iam_policy(@cloud_id)
687
+ if policy and policy.bindings
688
+ policy.bindings.each { |b|
689
+ if b.members.include?("allUsers") and b.role == "roles/cloudfunctions.invoker"
690
+ return true
691
+ end
692
+ }
693
+ end
694
+ false
695
+ end
696
+
697
+ def buildDesc(no_upload = false)
583
698
  labels = Hash[@tags.keys.map { |k|
584
699
  [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
585
700
  ]
@@ -631,6 +746,12 @@ module example.com/cloudfunction
631
746
  desc[:https_trigger] = MU::Cloud::Google.function(:HttpsTrigger).new
632
747
  end
633
748
 
749
+ if @config["internal_only"]
750
+ desc[:ingress_settings] = "ALLOW_INTERNAL_ONLY"
751
+ else
752
+ desc[:ingress_settings] = "ALLOW_ALL"
753
+ end
754
+
634
755
 
635
756
  if @config['environment_variable']
636
757
  @config['environment_variable'].each { |var|
@@ -658,7 +779,12 @@ module example.com/cloudfunction
658
779
  else
659
780
  MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
660
781
  end
661
- desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
782
+ bucket = MU::Cloud::Google.adminBucketName(credentials)
783
+
784
+ desc[:source_archive_url] = "gs://#{bucket}/#{@mu_name}-cloudfunction.zip"
785
+ if !no_upload
786
+ MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
787
+ end
662
788
 
663
789
  if tempfile
664
790
  tempfile.close
@@ -303,7 +303,8 @@ module MU
303
303
  bok['name'] = cloud_desc.project_id
304
304
  bok['cloud_id'] = cloud_desc.project_id
305
305
  # if cloud_desc.name != cloud_desc.project_id
306
- bok['display_name'] = cloud_desc.name
306
+ bok['display_name'] = cloud_desc.name
307
+ bok['display_name'] ||= ""
307
308
  # end
308
309
 
309
310
  if cloud_desc.parent and cloud_desc.parent.id