cloud-mu 3.5.0 → 3.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +5 -2
  3. data/Berksfile.lock +135 -0
  4. data/ansible/roles/mu-base/README.md +33 -0
  5. data/ansible/roles/mu-base/defaults/main.yml +2 -0
  6. data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
  7. data/ansible/roles/mu-base/files/check_apm.sh +18 -0
  8. data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
  9. data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
  10. data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
  11. data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
  12. data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
  13. data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
  14. data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
  15. data/ansible/roles/mu-base/files/logrotate.conf +35 -0
  16. data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
  17. data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
  18. data/ansible/roles/mu-base/handlers/main.yml +5 -0
  19. data/ansible/roles/mu-base/meta/main.yml +53 -0
  20. data/ansible/roles/mu-base/tasks/main.yml +113 -0
  21. data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
  22. data/ansible/roles/mu-base/tests/inventory +2 -0
  23. data/ansible/roles/mu-base/tests/test.yml +5 -0
  24. data/ansible/roles/mu-base/vars/main.yml +1 -0
  25. data/ansible/roles/mu-compliance/README.md +33 -0
  26. data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
  27. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
  28. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
  29. data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
  30. data/ansible/roles/mu-compliance/meta/main.yml +53 -0
  31. data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
  32. data/ansible/roles/mu-compliance/tests/inventory +2 -0
  33. data/ansible/roles/mu-compliance/tests/test.yml +5 -0
  34. data/ansible/roles/mu-compliance/vars/main.yml +4 -0
  35. data/ansible/roles/mu-elastic/README.md +51 -0
  36. data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
  37. data/ansible/roles/mu-elastic/files/jvm.options +93 -0
  38. data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
  39. data/ansible/roles/mu-elastic/meta/main.yml +52 -0
  40. data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
  41. data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
  42. data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
  43. data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
  44. data/ansible/roles/mu-elastic/tests/inventory +2 -0
  45. data/ansible/roles/mu-elastic/tests/test.yml +5 -0
  46. data/ansible/roles/mu-elastic/vars/main.yml +2 -0
  47. data/ansible/roles/mu-logstash/README.md +51 -0
  48. data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
  49. data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
  50. data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
  51. data/ansible/roles/mu-logstash/files/jvm.options +84 -0
  52. data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
  53. data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
  54. data/ansible/roles/mu-logstash/meta/main.yml +52 -0
  55. data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
  56. data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
  57. data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
  58. data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
  59. data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
  60. data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
  61. data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
  62. data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
  63. data/ansible/roles/mu-logstash/tests/inventory +2 -0
  64. data/ansible/roles/mu-logstash/tests/test.yml +5 -0
  65. data/ansible/roles/mu-logstash/vars/main.yml +2 -0
  66. data/ansible/roles/mu-rdp/README.md +33 -0
  67. data/ansible/roles/mu-rdp/meta/main.yml +53 -0
  68. data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
  69. data/ansible/roles/mu-rdp/tests/inventory +2 -0
  70. data/ansible/roles/mu-rdp/tests/test.yml +5 -0
  71. data/ansible/roles/mu-windows/tasks/main.yml +3 -0
  72. data/bin/mu-ansible-secret +1 -1
  73. data/bin/mu-aws-setup +4 -3
  74. data/bin/mu-azure-setup +5 -5
  75. data/bin/mu-configure +25 -17
  76. data/bin/mu-firewall-allow-clients +1 -0
  77. data/bin/mu-gcp-setup +3 -3
  78. data/bin/mu-load-config.rb +1 -0
  79. data/bin/mu-node-manage +66 -33
  80. data/bin/mu-self-update +2 -2
  81. data/bin/mu-upload-chef-artifacts +6 -1
  82. data/bin/mu-user-manage +1 -1
  83. data/cloud-mu.gemspec +25 -23
  84. data/cookbooks/firewall/CHANGELOG.md +417 -224
  85. data/cookbooks/firewall/LICENSE +202 -0
  86. data/cookbooks/firewall/README.md +153 -126
  87. data/cookbooks/firewall/TODO.md +6 -0
  88. data/cookbooks/firewall/attributes/firewalld.rb +7 -0
  89. data/cookbooks/firewall/attributes/iptables.rb +3 -3
  90. data/cookbooks/firewall/chefignore +115 -0
  91. data/cookbooks/firewall/libraries/helpers.rb +5 -0
  92. data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
  93. data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
  94. data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
  95. data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
  96. data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
  97. data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
  98. data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
  99. data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
  100. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
  101. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
  102. data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
  103. data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
  104. data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
  105. data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
  106. data/cookbooks/firewall/metadata.json +40 -1
  107. data/cookbooks/firewall/metadata.rb +15 -0
  108. data/cookbooks/firewall/recipes/default.rb +7 -7
  109. data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
  110. data/cookbooks/firewall/recipes/firewalld.rb +87 -0
  111. data/cookbooks/firewall/renovate.json +18 -0
  112. data/cookbooks/firewall/resources/firewalld.rb +28 -0
  113. data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
  114. data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
  115. data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
  116. data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
  117. data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
  118. data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
  119. data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
  120. data/cookbooks/firewall/resources/nftables.rb +71 -0
  121. data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
  122. data/cookbooks/mu-activedirectory/Berksfile +1 -1
  123. data/cookbooks/mu-activedirectory/metadata.rb +1 -1
  124. data/cookbooks/mu-firewall/metadata.rb +2 -2
  125. data/cookbooks/mu-master/Berksfile +4 -3
  126. data/cookbooks/mu-master/attributes/default.rb +5 -2
  127. data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
  128. data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
  129. data/cookbooks/mu-master/libraries/mu.rb +24 -0
  130. data/cookbooks/mu-master/metadata.rb +5 -5
  131. data/cookbooks/mu-master/recipes/default.rb +31 -20
  132. data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
  133. data/cookbooks/mu-master/recipes/init.rb +58 -19
  134. data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
  135. data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
  136. data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
  137. data/cookbooks/mu-php54/Berksfile +1 -1
  138. data/cookbooks/mu-php54/metadata.rb +2 -2
  139. data/cookbooks/mu-tools/Berksfile +2 -3
  140. data/cookbooks/mu-tools/attributes/default.rb +3 -4
  141. data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
  142. data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
  143. data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
  144. data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
  145. data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
  146. data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
  147. data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
  148. data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
  149. data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
  150. data/cookbooks/mu-tools/libraries/helper.rb +21 -9
  151. data/cookbooks/mu-tools/metadata.rb +4 -4
  152. data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
  153. data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
  154. data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
  155. data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
  156. data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
  157. data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
  158. data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
  159. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  160. data/data_bags/nagios_services/apm_backend_connect.json +5 -0
  161. data/data_bags/nagios_services/apm_listen.json +5 -0
  162. data/data_bags/nagios_services/elastic_shards.json +5 -0
  163. data/data_bags/nagios_services/logstash.json +5 -0
  164. data/data_bags/nagios_services/rhel7_updates.json +8 -0
  165. data/extras/image-generators/AWS/centos7.yaml +1 -0
  166. data/extras/image-generators/AWS/rhel7.yaml +21 -0
  167. data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
  168. data/extras/image-generators/AWS/win2k16.yaml +1 -0
  169. data/extras/image-generators/AWS/win2k19.yaml +1 -0
  170. data/extras/list-stock-amis +0 -0
  171. data/extras/ruby_rpm/muby.spec +8 -5
  172. data/extras/vault_tools/export_vaults.sh +1 -1
  173. data/extras/vault_tools/recreate_vaults.sh +0 -0
  174. data/extras/vault_tools/test_vaults.sh +0 -0
  175. data/install/deprecated-bash-library.sh +1 -1
  176. data/install/installer +4 -2
  177. data/modules/mommacat.ru +3 -1
  178. data/modules/mu/adoption.rb +1 -1
  179. data/modules/mu/cloud/dnszone.rb +2 -2
  180. data/modules/mu/cloud/machine_images.rb +26 -25
  181. data/modules/mu/cloud/resource_base.rb +213 -182
  182. data/modules/mu/cloud/server_pool.rb +1 -1
  183. data/modules/mu/cloud/ssh_sessions.rb +7 -5
  184. data/modules/mu/cloud/wrappers.rb +2 -2
  185. data/modules/mu/cloud.rb +1 -1
  186. data/modules/mu/config/bucket.rb +1 -1
  187. data/modules/mu/config/function.rb +6 -1
  188. data/modules/mu/config/loadbalancer.rb +24 -2
  189. data/modules/mu/config/ref.rb +12 -0
  190. data/modules/mu/config/role.rb +1 -1
  191. data/modules/mu/config/schema_helpers.rb +42 -9
  192. data/modules/mu/config/server.rb +43 -27
  193. data/modules/mu/config/tail.rb +19 -10
  194. data/modules/mu/config.rb +6 -5
  195. data/modules/mu/defaults/AWS.yaml +78 -114
  196. data/modules/mu/deploy.rb +9 -2
  197. data/modules/mu/groomer.rb +12 -4
  198. data/modules/mu/groomers/ansible.rb +104 -20
  199. data/modules/mu/groomers/chef.rb +15 -6
  200. data/modules/mu/master.rb +9 -4
  201. data/modules/mu/mommacat/daemon.rb +4 -2
  202. data/modules/mu/mommacat/naming.rb +1 -2
  203. data/modules/mu/mommacat/storage.rb +7 -2
  204. data/modules/mu/mommacat.rb +33 -6
  205. data/modules/mu/providers/aws/database.rb +161 -8
  206. data/modules/mu/providers/aws/dnszone.rb +11 -6
  207. data/modules/mu/providers/aws/endpoint.rb +81 -6
  208. data/modules/mu/providers/aws/firewall_rule.rb +254 -172
  209. data/modules/mu/providers/aws/function.rb +65 -3
  210. data/modules/mu/providers/aws/loadbalancer.rb +39 -28
  211. data/modules/mu/providers/aws/log.rb +2 -1
  212. data/modules/mu/providers/aws/role.rb +25 -7
  213. data/modules/mu/providers/aws/server.rb +36 -12
  214. data/modules/mu/providers/aws/server_pool.rb +237 -127
  215. data/modules/mu/providers/aws/storage_pool.rb +7 -1
  216. data/modules/mu/providers/aws/user.rb +1 -1
  217. data/modules/mu/providers/aws/userdata/linux.erb +6 -2
  218. data/modules/mu/providers/aws/userdata/windows.erb +7 -5
  219. data/modules/mu/providers/aws/vpc.rb +49 -25
  220. data/modules/mu/providers/aws.rb +13 -8
  221. data/modules/mu/providers/azure/container_cluster.rb +1 -1
  222. data/modules/mu/providers/azure/loadbalancer.rb +2 -2
  223. data/modules/mu/providers/azure/server.rb +5 -2
  224. data/modules/mu/providers/azure/userdata/linux.erb +1 -1
  225. data/modules/mu/providers/azure.rb +11 -8
  226. data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
  227. data/modules/mu/providers/google/container_cluster.rb +15 -2
  228. data/modules/mu/providers/google/folder.rb +2 -1
  229. data/modules/mu/providers/google/function.rb +130 -4
  230. data/modules/mu/providers/google/habitat.rb +2 -1
  231. data/modules/mu/providers/google/loadbalancer.rb +407 -160
  232. data/modules/mu/providers/google/role.rb +16 -3
  233. data/modules/mu/providers/google/server.rb +5 -1
  234. data/modules/mu/providers/google/user.rb +25 -18
  235. data/modules/mu/providers/google/userdata/linux.erb +1 -1
  236. data/modules/mu/providers/google/vpc.rb +53 -7
  237. data/modules/mu/providers/google.rb +39 -39
  238. data/modules/mu.rb +8 -8
  239. data/modules/tests/elk.yaml +46 -0
  240. data/test/mu-master-test/controls/all_in_one.rb +1 -1
  241. metadata +207 -112
  242. data/cookbooks/firewall/CONTRIBUTING.md +0 -2
  243. data/cookbooks/firewall/MAINTAINERS.md +0 -19
  244. data/cookbooks/firewall/libraries/matchers.rb +0 -30
  245. data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -114,20 +114,6 @@ module MU
114
114
  raise MuError, "VPC endpoint failed #{endpoint_id}: #{resp}" if resp.state == "failed"
115
115
  end
116
116
 
117
- if @config["enable_traffic_logging"]
118
- loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
119
- logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
120
-
121
- MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup.mu_name}"
122
- MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
123
- resource_ids: [@cloud_id],
124
- resource_type: "VPC",
125
- traffic_type: "ALL",
126
- log_group_name: loggroup.mu_name,
127
- deliver_logs_permission_arn: logrole.cloudobj.arn
128
- )
129
- end
130
-
131
117
  nat_gateways = create_subnets
132
118
 
133
119
  notify
@@ -270,6 +256,39 @@ module MU
270
256
  }
271
257
  end
272
258
 
259
+ if @config["enable_traffic_logging"]
260
+ ext = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_flow_logs(
261
+ filter: [
262
+ { name: "resource-id", values: [@cloud_id] }
263
+ ]
264
+ )
265
+ # XXX a smarter guard would filter with more specificity
266
+ if !ext or ext.flow_logs.empty?
267
+ loggroup = if @config['log_group_name']
268
+ @config['log_group_name']
269
+ else
270
+ @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs").mu_name
271
+ end
272
+ logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
273
+
274
+
275
+ MU.log "Enabling traffic logging on VPC #{@mu_name} to log group #{loggroup}"
276
+ MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).create_flow_logs(
277
+ resource_ids: [@cloud_id],
278
+ resource_type: "VPC",
279
+ traffic_type: "ALL",
280
+ log_group_name: loggroup,
281
+ deliver_logs_permission_arn: logrole.cloudobj.arn,
282
+ tag_specifications: [
283
+ {
284
+ resource_type: "vpc-flow-log",
285
+ tags: @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
286
+ }
287
+ ]
288
+ )
289
+ end
290
+ end
291
+
273
292
  end
274
293
 
275
294
  # Locate an existing VPC or VPCs and return an array containing matching AWS resource descriptors for those that match.
@@ -942,13 +961,15 @@ module MU
942
961
  ok = true
943
962
 
944
963
  if vpc["enable_traffic_logging"]
945
- logdesc = {
946
- "name" => vpc['name']+"loggroup",
947
- }
948
- logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
949
- # logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
950
- configurator.insertKitten(logdesc, "logs")
951
- MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
964
+ if !vpc['log_group_name']
965
+ logdesc = {
966
+ "name" => vpc['name']+"loggroup",
967
+ }
968
+ logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
969
+ # logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
970
+ configurator.insertKitten(logdesc, "logs")
971
+ MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
972
+ end
952
973
 
953
974
  roledesc = {
954
975
  "name" => vpc['name']+"logrole",
@@ -971,20 +992,23 @@ module MU
971
992
  "targets" => [
972
993
  {
973
994
  "type" => "log",
974
- "identifier" => vpc['name']+"loggroup"
995
+ "identifier" => vpc['log_group_name'] ? vpc['log_group_name'] : vpc['name']+"loggroup"
975
996
  }
976
997
  ]
977
998
  }
978
- ],
979
- "dependencies" => [
999
+ ]
1000
+ }
1001
+ if !vpc['log_group_name']
1002
+ roledesc["dependencies"] = [
980
1003
  {
981
1004
  "type" => "log",
982
1005
  "name" => vpc['name']+"loggroup"
983
1006
  }
984
1007
  ]
985
- }
1008
+ end
986
1009
  roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
987
1010
  roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
1011
+
988
1012
  configurator.insertKitten(roledesc, "roles")
989
1013
  MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
990
1014
  end
@@ -44,6 +44,12 @@ module MU
44
44
  [cfg['account_number']]
45
45
  end
46
46
 
47
+ # Cloud-specific resource methods or attributes we want exposed for
48
+ # reading by {MU::Cloud}
49
+ def self.customAttrReaders
50
+ [:region, :cloudformation_data]
51
+ end
52
+
47
53
  # A hook that is always called just before any of the instance method of
48
54
  # our resource implementations gets invoked, so that we can ensure that
49
55
  # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -51,10 +57,6 @@ module MU
51
57
  # @param cloudobj [MU::Cloud]
52
58
  # @param _deploy [MU::MommaCat]
53
59
  def self.resourceInitHook(cloudobj, _deploy)
54
- class << self
55
- attr_reader :cloudformation_data
56
- attr_reader :region
57
- end
58
60
  return if !cloudobj
59
61
  cloudobj.instance_variable_set(:@cloudformation_data, {})
60
62
 
@@ -409,8 +411,10 @@ end
409
411
  end
410
412
 
411
413
  # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
412
- # @param deploy_id [String]: The deploy for which we're writing the secret
414
+ # @param deploy [String]: The deploy for which we're writing the secret
413
415
  # @param value [String]: The contents of the secret
416
+ # @param name [String]: File/object name
417
+ # @param credentials [String]
414
418
  def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
415
419
  require "aws-sdk-s3"
416
420
  name ||= deploy.deploy_id+"-secret"
@@ -531,9 +535,9 @@ end
531
535
  if !@@is_in_aws.nil?
532
536
  return @@is_in_aws
533
537
  end
534
-
538
+ start = Time.now
535
539
  begin
536
- Timeout.timeout(4) do
540
+ Timeout.timeout(10) do
537
541
  instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
538
542
  if !instance_id.nil? and instance_id.size > 0
539
543
  @@is_in_aws = true
@@ -550,6 +554,7 @@ end
550
554
  end
551
555
  rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH
552
556
  end
557
+ MU.log "Fetch of http://169.254.169.254/latest/meta-data/instance-id took #{sprintf("%.2fs", Time.now-start)}", MU::WARN
553
558
 
554
559
  @@is_in_aws = false
555
560
  false
@@ -978,7 +983,7 @@ end
978
983
  id = matches.first
979
984
  elsif matches.size == 0
980
985
  if raise_on_missing
981
- raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
986
+ raise MuError, "No IAM or ACM certificate named #{name} was found in #{region} (credentials: #{(credentials.nil? or credentials.empty?) ? "default" : credentials})"
982
987
  else
983
988
  return nil
984
989
  end
@@ -249,7 +249,7 @@ module MU
249
249
  os_obj = MU::Cloud::Azure.containers(:ContainerServiceWindowsProfile, model_version: "V2019_02_01").new
250
250
  os_obj.admin_username = "muadmin"
251
251
  # Azure password constraints are extra-annoying
252
- winpass = MU.generateWindowsPassword(safe_pattern: '!@#$%^&*()', retries: 150)
252
+ winpass = MU.generatePassword(safe_pattern: '!@#$%^&*()', retries: 150)
253
253
  # TODO store this somewhere the user can get at it
254
254
  os_obj.admin_password = winpass
255
255
  os_obj
@@ -50,8 +50,8 @@ module MU
50
50
  # Register a Server node with an existing LoadBalancer.
51
51
  #
52
52
  # @param instance_id [String] A node to register.
53
- # @param targetgroups [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
54
- def registerNode(instance_id, targetgroups: nil)
53
+ # @param backends [Array<String>] The target group(s) of which this node should be made a member. Not applicable to classic LoadBalancers. If not supplied, the node will be registered to all available target groups on this LoadBalancer.
54
+ def registerTarget(instance_id, backends: nil)
55
55
  end
56
56
 
57
57
  # Does this resource type exist as a global (cloud-wide) artifact, or
@@ -426,7 +426,10 @@ MU.log "Azure::Server.find called", MU::NOTICE, details: args
426
426
  # @param size [String]: Size (in gb) of the new volume
427
427
  # @param type [String]: Cloud storage type of the volume, if applicable
428
428
  # @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
429
- def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
429
+ def addVolume(dev: nil, size: 0, type: "pd-standard", delete_on_termination: false)
430
+ if dev.nil? or size == 0
431
+ raise MuError, "Must specify a device name and a size for addVolume"
432
+ end
430
433
  end
431
434
 
432
435
  # Determine whether the node in question exists at the Cloud provider
@@ -785,7 +788,7 @@ MU.log "Azure::Server.find called", MU::NOTICE, details: args
785
788
  os_obj.admin_password = begin
786
789
  @deploy.fetchSecret(@mu_name, "windows_admin_password")
787
790
  rescue MU::MommaCat::SecretError
788
- pw = MU.generateWindowsPassword
791
+ pw = MU.generatePassword
789
792
  @deploy.saveNodeSecret(@mu_name, pw, "windows_admin_password")
790
793
  pw
791
794
  end
@@ -105,7 +105,7 @@ umask 0077
105
105
 
106
106
  # Install Chef now, because why not?
107
107
  if [ ! -f /opt/chef/embedded/bin/ruby ];then
108
- curl https://www.chef.io/chef/install.sh > chef-install.sh
108
+ curl https://omnitruck.chef.io/install.sh > chef-install.sh
109
109
  set +e
110
110
  # We may run afoul of a synchronous bootstrap process doing the same thing. So
111
111
  # wait until we've managed to run successfully.
@@ -52,6 +52,12 @@ module MU
52
52
  []
53
53
  end
54
54
 
55
+ # Cloud-specific resource methods or attributes we want exposed for
56
+ # reading by {MU::Cloud}
57
+ def self.customAttrReaders
58
+ [:region, :resource_group]
59
+ end
60
+
55
61
  # A hook that is always called just before any of the instance method of
56
62
  # our resource implementations gets invoked, so that we can ensure that
57
63
  # repetitive setup tasks (like resolving +:resource_group+ for Azure
@@ -59,9 +65,6 @@ module MU
59
65
  # @param cloudobj [MU::Cloud]
60
66
  # @param deploy [MU::MommaCat]
61
67
  def self.resourceInitHook(cloudobj, deploy)
62
- class << self
63
- attr_reader :resource_group
64
- end
65
68
  return if !cloudobj
66
69
 
67
70
  rg = if !deploy
@@ -565,12 +568,12 @@ MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
565
568
  return @@metadata if svc == "instance" and @@metadata
566
569
  base_url = "http://169.254.169.254/metadata/#{svc}"
567
570
  args["api-version"] = api_version
568
- arg_str = args.keys.map { |k| k.to_s+"="+args[k].to_s }.join("&")
571
+ arg_str = args.keys.sort.map { |k| k.to_s+"="+CGI.escape(args[k].to_s) }.join("&")
569
572
 
570
573
  begin
571
574
  Timeout.timeout(2) do
572
- resp = JSON.parse(URI.open("#{base_url}/?#{arg_str}","Metadata"=>"true").read)
573
- MU.log "curl -H Metadata:true "+"#{base_url}/?#{arg_str}", loglevel, details: resp
575
+ resp = JSON.parse(URI.open("#{base_url}?#{arg_str}","Metadata"=>"true").read)
576
+ MU.log "curl -H Metadata:true "+"#{base_url}?#{arg_str}", loglevel, details: resp
574
577
  if svc != "instance"
575
578
  return resp
576
579
  else
@@ -597,9 +600,9 @@ MU.log "vault existence check #{vaultname}", MU::WARN, details: resp
597
600
  cfg = credConfig(credentials)
598
601
 
599
602
  if cfg and MU::Cloud::Azure.hosted?
600
- token = MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2018-02-01", args: { "resource"=>"https://management.azure.com/" })
603
+ token = MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2020-09-01", args: { "resource"=>"https://management.azure.com/" })
601
604
  if !token
602
- MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2018-02-01", args: { "resource"=>"https://management.azure.com/" }, debug: true)
605
+ MU::Cloud::Azure.get_metadata("identity/oauth2/token", "2020-09-01", args: { "resource"=>"https://management.azure.com/" }, debug: true)
603
606
  raise MuError, "Failed to get machine oauth token"
604
607
  end
605
608
  machine = MU::Cloud::Azure.get_metadata
@@ -248,7 +248,7 @@ module MU
248
248
  end
249
249
  # Placeholder. This is a NOOP for CloudFormation, which doesn't build
250
250
  # resources directly.
251
- def self.genericMuDNSEntry(*args)
251
+ def self.genericMuDNSEntry(**args)
252
252
  MU.log "find() not implemented for CloudFormation layer", MU::DEBUG
253
253
  nil
254
254
  end
@@ -95,7 +95,7 @@ module MU
95
95
  desc = {
96
96
  :name => @mu_name.downcase,
97
97
  :description => @deploy.deploy_id,
98
- :network => @vpc.cloud_id,
98
+ :network => @vpc.url,
99
99
  :enable_tpu => @config['tpu'],
100
100
  :resource_labels => labels,
101
101
  :locations => locations,
@@ -108,6 +108,7 @@ module MU
108
108
  ),
109
109
  }
110
110
 
111
+
111
112
  if @config['kubernetes']
112
113
  desc[:addons_config] = MU::Cloud::Google.container(:AddonsConfig).new(
113
114
  horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
@@ -135,6 +136,8 @@ module MU
135
136
  end
136
137
  }
137
138
  end
139
+
140
+
138
141
  if @config['log_facility'] == "kubernetes"
139
142
  desc[:logging_service] = "logging.googleapis.com/kubernetes"
140
143
  desc[:monitoring_service] = "monitoring.googleapis.com/kubernetes"
@@ -183,6 +186,7 @@ module MU
183
186
  )
184
187
  end
185
188
 
189
+
186
190
  if @config['ip_aliases'] or @config['custom_subnet'] or
187
191
  @config['services_ip_block'] or @config['services_ip_block_name'] or
188
192
  @config['pod_ip_block'] or @config['pod_ip_block_name'] or
@@ -210,17 +214,26 @@ module MU
210
214
  end
211
215
 
212
216
  if @config['services_ip_block']
217
+ if @vpc.project_id != @project_id
218
+ alloc_desc[:services_secondary_range_name] ||= @config['name']+"-services"
219
+ @vpc.addSecondaryRange(desc[:subnetwork], @config['services_ip_block'], alloc_desc[:services_secondary_range_name])
220
+
221
+ end
213
222
  alloc_desc[:services_ipv4_cidr_block] = @config['services_ip_block']
214
223
  end
215
224
  if @config['tpu_ip_block']
216
225
  alloc_desc[:tpu_ipv4_cidr_block] = @config['tpu_ip_block']
217
226
  end
218
227
  if @config['pod_ip_block']
228
+ if @vpc.project_id != @project_id
229
+ alloc_desc[:cluster_secondary_range_name] ||= @config['name']+"-pods"
230
+ @vpc.addSecondaryRange(desc[:subnetwork], @config['pod_ip_block'], alloc_desc[:cluster_secondary_range_name])
231
+
232
+ end
219
233
  alloc_desc[:cluster_ipv4_cidr_block] = @config['pod_ip_block']
220
234
  end
221
235
 
222
236
  desc[:ip_allocation_policy] = MU::Cloud::Google.container(:IpAllocationPolicy).new(alloc_desc)
223
- pp alloc_desc
224
237
  end
225
238
 
226
239
  if @config['authorized_networks'] and @config['authorized_networks'].size > 0
@@ -260,7 +260,8 @@ module MU
260
260
  args[:flags]['parent_id']
261
261
  else
262
262
  my_org = MU::Cloud::Google.getOrg(args[:credentials])
263
- my_org.name
263
+ # XXX re-raise with a clear permission error
264
+ my_org.name if my_org
264
265
  end
265
266
 
266
267
  if args[:cloud_id]
@@ -120,7 +120,7 @@ module example.com/cloudfunction
120
120
  def groom
121
121
  desc = {}
122
122
 
123
- func_obj = buildDesc
123
+ func_obj = buildDesc(true)
124
124
 
125
125
  labels = Hash[@tags.keys.map { |k|
126
126
  [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
@@ -161,6 +161,13 @@ module example.com/cloudfunction
161
161
  cloud_desc.event_trigger.resource != @config['triggers'].first['resource']
162
162
  need_update = true
163
163
  end
164
+ elsif !cloud_desc.https_trigger
165
+ need_update = true
166
+ end
167
+
168
+ if (@config['internal_only'] and cloud_desc.ingress_settings != "ALLOW_INTERNAL_ONLY") or
169
+ (!@config['internal_only'] and cloud_desc.ingress_settings != "ALLOW_ALL")
170
+ need_update = true
164
171
  end
165
172
 
166
173
  current = Dir.mktmpdir(@mu_name+"-current") { |dir|
@@ -206,6 +213,7 @@ module example.com/cloudfunction
206
213
  end
207
214
 
208
215
  if need_update
216
+ MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
209
217
  MU.log "Updating Cloud Function #{@cloud_id}", MU::NOTICE, details: func_obj
210
218
  begin
211
219
  MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
@@ -235,6 +243,71 @@ module example.com/cloudfunction
235
243
  tempfile.unlink
236
244
  end
237
245
 
246
+ policy = MU::Cloud::Google.function(credentials: @credentials).get_project_location_function_iam_policy(@cloud_id)
247
+
248
+ if @config['allow_unauthenticated'] and !allowsUnauthencated?
249
+ policy ||= MU::Cloud::Google.function(:Policy).new(
250
+ bindings: []
251
+ )
252
+ policy.bindings ||= []
253
+ policy.bindings << MU::Cloud::Google.function(:Binding).new(
254
+ members: ["allUsers"],
255
+ role: "roles/cloudfunctions.invoker"
256
+ )
257
+
258
+ pol_req = MU::Cloud::Google.function(:SetIamPolicyRequest).new(
259
+ policy: policy
260
+ )
261
+ MU.log "Enabling anonymous invocation of Cloud Function #{@mu_name}", MU::NOTICE
262
+ MU::Cloud::Google.function(credentials: @credentials).set_function_iam_policy(@cloud_id, pol_req)
263
+ elsif !@config['allow_unauthenticated'] and allowsUnauthencated?
264
+ policy.bindings.reject! { |b|
265
+ b.members.include?("allUsers") and b.role == "roles/cloudfunctions.invoker"
266
+ }
267
+ pol_req = MU::Cloud::Google.function(:SetIamPolicyRequest).new(
268
+ policy: policy
269
+ )
270
+ MU.log "Disabling anonymous invocation of Cloud Function #{@mu_name}", MU::NOTICE
271
+ MU::Cloud::Google.function(credentials: @credentials).set_function_iam_policy(@cloud_id, pol_req)
272
+ end
273
+
274
+ # If we have a loadbalancer configured, attach us to it
275
+ if !@config['loadbalancers'].nil?
276
+ if @loadbalancers.nil?
277
+ raise MuError, "#{@mu_name} is configured to use LoadBalancers, but none have been loaded by dependencies()"
278
+ end
279
+
280
+ neg_name = @deploy.getResourceName(@config["name"], max_length: 19, never_gen_unique: true).downcase
281
+ neg_desc = begin
282
+ MU::Cloud::Google.compute(credentials: @config['credentials']).get_region_network_endpoint_group(@project_id, @config['region'], neg_name)
283
+ rescue ::Google::Apis::ClientError => e
284
+ raise e if e.message !~ /notFound:/
285
+ neg_obj = MU::Cloud::Google.compute(:NetworkEndpointGroup).new(
286
+ name: neg_name,
287
+ description: @deploy.deploy_id,
288
+ cloud_function: MU::Cloud::Google.compute(:NetworkEndpointGroupCloudFunction).new(
289
+ function: @cloud_id.gsub(/.*?\//, '')
290
+ ),
291
+ network_endpoint_type: "SERVERLESS"
292
+ )
293
+ MU.log "Creating Network Endpoint Group #{neg_name}", details: neg_obj
294
+ MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_network_endpoint_group(@project_id, @config['region'], neg_obj)
295
+ retry
296
+ end
297
+
298
+ @loadbalancers.each { |lb|
299
+ # if !lb.targetgroups
300
+ # MU.retrier([], max: 6, wait: 15, loop_if: Proc.new { !lb.targetgroups }) {
301
+ # lb.cloud_desc(use_cache: false)
302
+ # }
303
+ # end
304
+ # lb.targetgroups.each_pair { |tg_name, tg|
305
+ # addTrigger(tg.target_group_arn, "elasticloadbalancing", tg_name)
306
+ # }
307
+ lb.registerTarget(neg_desc.self_link)
308
+ }
309
+ end
310
+
238
311
  end
239
312
 
240
313
  # Return the metadata for this project's configuration
@@ -330,6 +403,12 @@ module example.com/cloudfunction
330
403
  bok["handler"] = cloud_desc.entry_point
331
404
  bok["timeout"] = cloud_desc.timeout.gsub(/[^\d]/, '').to_i
332
405
 
406
+ if cloud_desc.ingress_settings and cloud_desc.ingress_settings == "ALLOW_INTERNAL_ONLY"
407
+ bok['internal_only'] = true
408
+ else
409
+ bok['internal_only'] = false
410
+ end
411
+
333
412
  if cloud_desc.vpc_connector
334
413
  bok["vpc_connector"] = cloud_desc.vpc_connector
335
414
  elsif cloud_desc.network
@@ -350,7 +429,7 @@ module example.com/cloudfunction
350
429
  type: "vpcs"
351
430
  )
352
431
  end
353
-
432
+
354
433
  if cloud_desc.environment_variables and cloud_desc.environment_variables.size > 0
355
434
  bok['environment_variable'] = cloud_desc.environment_variables.keys.map { |k| { "key" => k, "value" => cloud_desc.environment_variables[k] } }
356
435
  end
@@ -373,6 +452,9 @@ module example.com/cloudfunction
373
452
  'zip_file' => codefile
374
453
  }
375
454
 
455
+
456
+ bok['allow_unauthenticated'] = allowsUnauthencated?
457
+
376
458
  bok
377
459
  end
378
460
 
@@ -411,6 +493,16 @@ module example.com/cloudfunction
411
493
  "type" => "string",
412
494
  "description" => "+DEPRECATED+ VPC Connector to attach, of the form +projects/my-project/locations/some-region/connectors/my-connector+. This option will be removed once proper google-cloud-sdk support for VPC Connectors becomes available, at which point we will piggyback on the normal +vpc+ stanza and resolve connectors as needed."
413
495
  },
496
+ "allow_unauthenticated" => {
497
+ "type" => "boolean",
498
+ "default" => false,
499
+ "description" => "Only applicable for HTTPS-triggered functions; allows function invocation without credentials"
500
+ },
501
+ "internal_only" => {
502
+ "type" => "boolean",
503
+ "default" => false,
504
+ "description" => "Permit only traffic from VPC networks in the same project or VPC SC perimeter"
505
+ },
414
506
  "vpc_connector_allow_all_egress" => {
415
507
  "type" => "boolean",
416
508
  "default" => false,
@@ -473,11 +565,13 @@ module example.com/cloudfunction
473
565
  # @return [String]: The Cloud Storage URL to the result
474
566
  def self.uploadPackage(zipfile, filename, credentials: nil)
475
567
  bucket = MU::Cloud::Google.adminBucketName(credentials)
568
+
476
569
  obj_obj = MU::Cloud::Google.storage(:Object).new(
477
570
  content_type: "application/zip",
478
571
  name: filename
479
572
  )
480
573
 
574
+ MU.log "Uploading #{zipfile} to #{bucket}/#{filename}"
481
575
  MU::Cloud::Google.storage(credentials: credentials).insert_object(
482
576
  bucket,
483
577
  obj_obj,
@@ -574,12 +668,33 @@ module example.com/cloudfunction
574
668
  # ok = false
575
669
  # end
576
670
 
671
+ if !function["loadbalancers"].nil?
672
+ function["loadbalancers"].each { |lb|
673
+ lb["name"] ||= lb["concurrent_load_balancer"]
674
+ if lb["name"]
675
+ MU::Config.addDependency(function, lb["name"], "loadbalancer")
676
+ end
677
+ }
678
+ end
679
+
577
680
  ok
578
681
  end
579
682
 
580
683
  private
581
684
 
582
- def buildDesc
685
+ def allowsUnauthencated?
686
+ policy = MU::Cloud::Google.function(credentials: @credentials).get_project_location_function_iam_policy(@cloud_id)
687
+ if policy and policy.bindings
688
+ policy.bindings.each { |b|
689
+ if b.members.include?("allUsers") and b.role == "roles/cloudfunctions.invoker"
690
+ return true
691
+ end
692
+ }
693
+ end
694
+ false
695
+ end
696
+
697
+ def buildDesc(no_upload = false)
583
698
  labels = Hash[@tags.keys.map { |k|
584
699
  [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
585
700
  ]
@@ -631,6 +746,12 @@ module example.com/cloudfunction
631
746
  desc[:https_trigger] = MU::Cloud::Google.function(:HttpsTrigger).new
632
747
  end
633
748
 
749
+ if @config["internal_only"]
750
+ desc[:ingress_settings] = "ALLOW_INTERNAL_ONLY"
751
+ else
752
+ desc[:ingress_settings] = "ALLOW_ALL"
753
+ end
754
+
634
755
 
635
756
  if @config['environment_variable']
636
757
  @config['environment_variable'].each { |var|
@@ -658,7 +779,12 @@ module example.com/cloudfunction
658
779
  else
659
780
  MU.log "#{@mu_name} using code packaged at #{@config['code']['zip_file']}"
660
781
  end
661
- desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
782
+ bucket = MU::Cloud::Google.adminBucketName(credentials)
783
+
784
+ desc[:source_archive_url] = "gs://#{bucket}/#{@mu_name}-cloudfunction.zip"
785
+ if !no_upload
786
+ MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
787
+ end
662
788
 
663
789
  if tempfile
664
790
  tempfile.close
@@ -303,7 +303,8 @@ module MU
303
303
  bok['name'] = cloud_desc.project_id
304
304
  bok['cloud_id'] = cloud_desc.project_id
305
305
  # if cloud_desc.name != cloud_desc.project_id
306
- bok['display_name'] = cloud_desc.name
306
+ bok['display_name'] = cloud_desc.name
307
+ bok['display_name'] ||= ""
307
308
  # end
308
309
 
309
310
  if cloud_desc.parent and cloud_desc.parent.id