cloud-mu 3.5.0 → 3.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Berksfile +5 -2
- data/Berksfile.lock +135 -0
- data/ansible/roles/mu-base/README.md +33 -0
- data/ansible/roles/mu-base/defaults/main.yml +2 -0
- data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
- data/ansible/roles/mu-base/files/check_apm.sh +18 -0
- data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
- data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
- data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
- data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
- data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
- data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
- data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
- data/ansible/roles/mu-base/files/logrotate.conf +35 -0
- data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
- data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
- data/ansible/roles/mu-base/handlers/main.yml +5 -0
- data/ansible/roles/mu-base/meta/main.yml +53 -0
- data/ansible/roles/mu-base/tasks/main.yml +113 -0
- data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
- data/ansible/roles/mu-base/tests/inventory +2 -0
- data/ansible/roles/mu-base/tests/test.yml +5 -0
- data/ansible/roles/mu-base/vars/main.yml +1 -0
- data/ansible/roles/mu-compliance/README.md +33 -0
- data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
- data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
- data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
- data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
- data/ansible/roles/mu-compliance/meta/main.yml +53 -0
- data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
- data/ansible/roles/mu-compliance/tests/inventory +2 -0
- data/ansible/roles/mu-compliance/tests/test.yml +5 -0
- data/ansible/roles/mu-compliance/vars/main.yml +4 -0
- data/ansible/roles/mu-elastic/README.md +51 -0
- data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
- data/ansible/roles/mu-elastic/files/jvm.options +93 -0
- data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
- data/ansible/roles/mu-elastic/meta/main.yml +52 -0
- data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
- data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
- data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
- data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
- data/ansible/roles/mu-elastic/tests/inventory +2 -0
- data/ansible/roles/mu-elastic/tests/test.yml +5 -0
- data/ansible/roles/mu-elastic/vars/main.yml +2 -0
- data/ansible/roles/mu-logstash/README.md +51 -0
- data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
- data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
- data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
- data/ansible/roles/mu-logstash/files/jvm.options +84 -0
- data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
- data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
- data/ansible/roles/mu-logstash/meta/main.yml +52 -0
- data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
- data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
- data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
- data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
- data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
- data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
- data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
- data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
- data/ansible/roles/mu-logstash/tests/inventory +2 -0
- data/ansible/roles/mu-logstash/tests/test.yml +5 -0
- data/ansible/roles/mu-logstash/vars/main.yml +2 -0
- data/ansible/roles/mu-rdp/README.md +33 -0
- data/ansible/roles/mu-rdp/meta/main.yml +53 -0
- data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
- data/ansible/roles/mu-rdp/tests/inventory +2 -0
- data/ansible/roles/mu-rdp/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/tasks/main.yml +3 -0
- data/bin/mu-ansible-secret +1 -1
- data/bin/mu-aws-setup +4 -3
- data/bin/mu-azure-setup +5 -5
- data/bin/mu-configure +25 -17
- data/bin/mu-firewall-allow-clients +1 -0
- data/bin/mu-gcp-setup +3 -3
- data/bin/mu-load-config.rb +1 -0
- data/bin/mu-node-manage +66 -33
- data/bin/mu-self-update +2 -2
- data/bin/mu-upload-chef-artifacts +6 -1
- data/bin/mu-user-manage +1 -1
- data/cloud-mu.gemspec +25 -23
- data/cookbooks/firewall/CHANGELOG.md +417 -224
- data/cookbooks/firewall/LICENSE +202 -0
- data/cookbooks/firewall/README.md +153 -126
- data/cookbooks/firewall/TODO.md +6 -0
- data/cookbooks/firewall/attributes/firewalld.rb +7 -0
- data/cookbooks/firewall/attributes/iptables.rb +3 -3
- data/cookbooks/firewall/chefignore +115 -0
- data/cookbooks/firewall/libraries/helpers.rb +5 -0
- data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
- data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
- data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
- data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
- data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
- data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
- data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
- data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
- data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
- data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
- data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
- data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
- data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
- data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
- data/cookbooks/firewall/metadata.json +40 -1
- data/cookbooks/firewall/metadata.rb +15 -0
- data/cookbooks/firewall/recipes/default.rb +7 -7
- data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
- data/cookbooks/firewall/recipes/firewalld.rb +87 -0
- data/cookbooks/firewall/renovate.json +18 -0
- data/cookbooks/firewall/resources/firewalld.rb +28 -0
- data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
- data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
- data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
- data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
- data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
- data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
- data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
- data/cookbooks/firewall/resources/nftables.rb +71 -0
- data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
- data/cookbooks/mu-activedirectory/Berksfile +1 -1
- data/cookbooks/mu-activedirectory/metadata.rb +1 -1
- data/cookbooks/mu-firewall/metadata.rb +2 -2
- data/cookbooks/mu-master/Berksfile +4 -3
- data/cookbooks/mu-master/attributes/default.rb +5 -2
- data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
- data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
- data/cookbooks/mu-master/libraries/mu.rb +24 -0
- data/cookbooks/mu-master/metadata.rb +5 -5
- data/cookbooks/mu-master/recipes/default.rb +31 -20
- data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
- data/cookbooks/mu-master/recipes/init.rb +58 -19
- data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
- data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
- data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
- data/cookbooks/mu-php54/Berksfile +1 -1
- data/cookbooks/mu-php54/metadata.rb +2 -2
- data/cookbooks/mu-tools/Berksfile +2 -3
- data/cookbooks/mu-tools/attributes/default.rb +3 -4
- data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
- data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
- data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
- data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
- data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
- data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
- data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
- data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
- data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
- data/cookbooks/mu-tools/libraries/helper.rb +21 -9
- data/cookbooks/mu-tools/metadata.rb +4 -4
- data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
- data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
- data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
- data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
- data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
- data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
- data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
- data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
- data/data_bags/nagios_services/apm_backend_connect.json +5 -0
- data/data_bags/nagios_services/apm_listen.json +5 -0
- data/data_bags/nagios_services/elastic_shards.json +5 -0
- data/data_bags/nagios_services/logstash.json +5 -0
- data/data_bags/nagios_services/rhel7_updates.json +8 -0
- data/extras/image-generators/AWS/centos7.yaml +1 -0
- data/extras/image-generators/AWS/rhel7.yaml +21 -0
- data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
- data/extras/image-generators/AWS/win2k16.yaml +1 -0
- data/extras/image-generators/AWS/win2k19.yaml +1 -0
- data/extras/list-stock-amis +0 -0
- data/extras/ruby_rpm/muby.spec +8 -5
- data/extras/vault_tools/export_vaults.sh +1 -1
- data/extras/vault_tools/recreate_vaults.sh +0 -0
- data/extras/vault_tools/test_vaults.sh +0 -0
- data/install/deprecated-bash-library.sh +1 -1
- data/install/installer +4 -2
- data/modules/mommacat.ru +3 -1
- data/modules/mu/adoption.rb +1 -1
- data/modules/mu/cloud/dnszone.rb +2 -2
- data/modules/mu/cloud/machine_images.rb +26 -25
- data/modules/mu/cloud/resource_base.rb +213 -182
- data/modules/mu/cloud/server_pool.rb +1 -1
- data/modules/mu/cloud/ssh_sessions.rb +7 -5
- data/modules/mu/cloud/wrappers.rb +2 -2
- data/modules/mu/cloud.rb +1 -1
- data/modules/mu/config/bucket.rb +1 -1
- data/modules/mu/config/function.rb +6 -1
- data/modules/mu/config/loadbalancer.rb +24 -2
- data/modules/mu/config/ref.rb +12 -0
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +42 -9
- data/modules/mu/config/server.rb +43 -27
- data/modules/mu/config/tail.rb +19 -10
- data/modules/mu/config.rb +6 -5
- data/modules/mu/defaults/AWS.yaml +78 -114
- data/modules/mu/deploy.rb +9 -2
- data/modules/mu/groomer.rb +12 -4
- data/modules/mu/groomers/ansible.rb +104 -20
- data/modules/mu/groomers/chef.rb +15 -6
- data/modules/mu/master.rb +9 -4
- data/modules/mu/mommacat/daemon.rb +4 -2
- data/modules/mu/mommacat/naming.rb +1 -2
- data/modules/mu/mommacat/storage.rb +7 -2
- data/modules/mu/mommacat.rb +33 -6
- data/modules/mu/providers/aws/database.rb +161 -8
- data/modules/mu/providers/aws/dnszone.rb +11 -6
- data/modules/mu/providers/aws/endpoint.rb +81 -6
- data/modules/mu/providers/aws/firewall_rule.rb +254 -172
- data/modules/mu/providers/aws/function.rb +65 -3
- data/modules/mu/providers/aws/loadbalancer.rb +39 -28
- data/modules/mu/providers/aws/log.rb +2 -1
- data/modules/mu/providers/aws/role.rb +25 -7
- data/modules/mu/providers/aws/server.rb +36 -12
- data/modules/mu/providers/aws/server_pool.rb +237 -127
- data/modules/mu/providers/aws/storage_pool.rb +7 -1
- data/modules/mu/providers/aws/user.rb +1 -1
- data/modules/mu/providers/aws/userdata/linux.erb +6 -2
- data/modules/mu/providers/aws/userdata/windows.erb +7 -5
- data/modules/mu/providers/aws/vpc.rb +49 -25
- data/modules/mu/providers/aws.rb +13 -8
- data/modules/mu/providers/azure/container_cluster.rb +1 -1
- data/modules/mu/providers/azure/loadbalancer.rb +2 -2
- data/modules/mu/providers/azure/server.rb +5 -2
- data/modules/mu/providers/azure/userdata/linux.erb +1 -1
- data/modules/mu/providers/azure.rb +11 -8
- data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
- data/modules/mu/providers/google/container_cluster.rb +15 -2
- data/modules/mu/providers/google/folder.rb +2 -1
- data/modules/mu/providers/google/function.rb +130 -4
- data/modules/mu/providers/google/habitat.rb +2 -1
- data/modules/mu/providers/google/loadbalancer.rb +407 -160
- data/modules/mu/providers/google/role.rb +16 -3
- data/modules/mu/providers/google/server.rb +5 -1
- data/modules/mu/providers/google/user.rb +25 -18
- data/modules/mu/providers/google/userdata/linux.erb +1 -1
- data/modules/mu/providers/google/vpc.rb +53 -7
- data/modules/mu/providers/google.rb +39 -39
- data/modules/mu.rb +8 -8
- data/modules/tests/elk.yaml +46 -0
- data/test/mu-master-test/controls/all_in_one.rb +1 -1
- metadata +207 -112
- data/cookbooks/firewall/CONTRIBUTING.md +0 -2
- data/cookbooks/firewall/MAINTAINERS.md +0 -19
- data/cookbooks/firewall/libraries/matchers.rb +0 -30
- data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -564,7 +564,12 @@ module MU
|
|
564
564
|
canned = Hash[MU::Cloud::Google.iam(credentials: args[:credentials]).list_roles.roles.map { |r| [r.name, r] }]
|
565
565
|
begin
|
566
566
|
MU::Cloud.resourceClass("Google", "Habitat").bindings(args[:project], credentials: args[:credentials]).each { |binding|
|
567
|
-
|
567
|
+
if binding.role =~ /^roles\//
|
568
|
+
found[binding.role] = canned[binding.role]
|
569
|
+
elsif binding.role =~ /^#{Regexp.quote(my_org.name)}\/roles\//
|
570
|
+
found[binding.role] = MU::Cloud::Google.iam(credentials: args[:credentials]).get_organization_role(binding.role)
|
571
|
+
end
|
572
|
+
|
568
573
|
}
|
569
574
|
rescue ::Google::Apis::ClientError => e
|
570
575
|
raise e if !e.message.match(/forbidden: /)
|
@@ -663,7 +668,6 @@ module MU
|
|
663
668
|
"credentials" => @config['credentials'],
|
664
669
|
"cloud_id" => @cloud_id
|
665
670
|
}
|
666
|
-
|
667
671
|
my_org = MU::Cloud::Google.getOrg(@config['credentials'])
|
668
672
|
|
669
673
|
# This can happen if the role_source isn't set correctly. This logic
|
@@ -680,6 +684,9 @@ module MU
|
|
680
684
|
|
681
685
|
# GSuite or Cloud Identity role
|
682
686
|
if cloud_desc.class == ::Google::Apis::AdminDirectoryV1::Role
|
687
|
+
if @cloud_id =~ /ncbi_snapshot_manager/
|
688
|
+
MU.log "directory-tier role", MU::NOTICE
|
689
|
+
end
|
683
690
|
return nil if cloud_desc.is_system_role
|
684
691
|
|
685
692
|
bok["name"] = @config['name'].gsub(/[^a-z0-9]/i, '-').downcase
|
@@ -702,6 +709,9 @@ module MU
|
|
702
709
|
bok['import'].sort! # at least be legible
|
703
710
|
end
|
704
711
|
else # otherwise it's a GCP IAM role of some kind
|
712
|
+
if @cloud_id =~ /ncbi_snapshot_manager/
|
713
|
+
MU.log "cloud-tier role", MU::NOTICE
|
714
|
+
end
|
705
715
|
|
706
716
|
return nil if cloud_desc.stage == "DISABLED"
|
707
717
|
if cloud_desc.name.match(/^roles\/([^\/]+)$/)
|
@@ -811,6 +821,9 @@ module MU
|
|
811
821
|
# to bother with them.
|
812
822
|
if bok['role_source'] == "canned" and
|
813
823
|
(bok['bindings'].nil? or bok['bindings'].empty?)
|
824
|
+
if @cloud_id =~ /ncbi_snapshot_manager/
|
825
|
+
MU.log "ditching at canned role check", MU::NOTICE
|
826
|
+
end
|
814
827
|
return nil
|
815
828
|
end
|
816
829
|
|
@@ -940,7 +953,7 @@ module MU
|
|
940
953
|
}
|
941
954
|
rescue ::Google::Apis::ClientError => e
|
942
955
|
if e.message.match(/forbidden: /)
|
943
|
-
MU.log "Do not have permissions to retrieve bindings in project #{project}, skipping", MU::WARN
|
956
|
+
MU.log "Do not have permissions to retrieve bindings in project #{project}, org #{MU::Cloud::Google.getOrg(credentials).display_name}, skipping", MU::WARN
|
944
957
|
else
|
945
958
|
raise e
|
946
959
|
end
|
@@ -1109,7 +1109,11 @@ next if !create
|
|
1109
1109
|
# @param size [String]: Size (in gb) of the new volume
|
1110
1110
|
# @param type [String]: Cloud storage type of the volume, if applicable
|
1111
1111
|
# @param delete_on_termination [Boolean]: Value of delete_on_termination flag to set
|
1112
|
-
def addVolume(dev, size, type: "pd-standard", delete_on_termination: false)
|
1112
|
+
def addVolume(dev: nil, size: 0, type: "pd-standard", delete_on_termination: false)
|
1113
|
+
if dev.nil? or size == 0
|
1114
|
+
raise MuError, "Must specify a device name and a size for addVolume"
|
1115
|
+
end
|
1116
|
+
|
1113
1117
|
devname = dev.gsub(/.*?\/([^\/]+)$/, '\1')
|
1114
1118
|
resname = MU::Cloud::Google.nameStr(@mu_name+"-"+devname)
|
1115
1119
|
MU.log "Creating disk #{resname}"
|
@@ -114,7 +114,7 @@ module MU
|
|
114
114
|
primary_email: @config['email'],
|
115
115
|
suspended: @config['suspend'],
|
116
116
|
is_admin: @config['admin'],
|
117
|
-
password: MU.
|
117
|
+
password: MU.generatePassword,
|
118
118
|
change_password_at_next_login: (@config.has_key?('force_password_change') ? @config['force_password_change'] : true)
|
119
119
|
)
|
120
120
|
|
@@ -310,6 +310,10 @@ module MU
|
|
310
310
|
end
|
311
311
|
end
|
312
312
|
|
313
|
+
# clock projects we can't list so we don't waste time trying them over
|
314
|
+
# and over
|
315
|
+
@@cant_list = []
|
316
|
+
|
313
317
|
# Locate and return cloud provider descriptors of this resource type
|
314
318
|
# which match the provided parameters, or all visible resources if no
|
315
319
|
# filters are specified. At minimum, implementations of +find+ must
|
@@ -341,24 +345,27 @@ module MU
|
|
341
345
|
|
342
346
|
if args[:project]
|
343
347
|
# project-local service accounts
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
348
|
+
if !@@cant_list.include?(args[:project])
|
349
|
+
resp = begin
|
350
|
+
MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
|
351
|
+
"projects/"+args[:project]
|
352
|
+
)
|
353
|
+
rescue ::Google::Apis::ClientError
|
354
|
+
@@cant_list << args[:project]
|
355
|
+
MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}, org #{MU::Cloud::Google.getOrg(args[:credentials]).display_name}", MU::WARN
|
356
|
+
end
|
351
357
|
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
358
|
+
if resp and resp.accounts
|
359
|
+
resp.accounts.each { |sa|
|
360
|
+
if args[:flags] and args[:flags]["skip_provider_owned"] and
|
361
|
+
MU::Cloud::Google::User.cannedServiceAcctName?(sa.name)
|
362
|
+
next
|
363
|
+
end
|
364
|
+
if !args[:cloud_id] or (sa.display_name and sa.display_name == args[:cloud_id]) or (sa.name and sa.name == args[:cloud_id]) or (sa.email and sa.email == args[:cloud_id])
|
365
|
+
found[sa.name] = sa
|
366
|
+
end
|
367
|
+
}
|
368
|
+
end
|
362
369
|
end
|
363
370
|
else
|
364
371
|
if cred_cfg['masquerade_as']
|
@@ -105,7 +105,7 @@ umask 0077
|
|
105
105
|
|
106
106
|
# Install Chef now, because why not?
|
107
107
|
if [ ! -f /opt/chef/embedded/bin/ruby ];then
|
108
|
-
curl https://
|
108
|
+
curl https://omnitruck.chef.io/install.sh > chef-install.sh
|
109
109
|
set +e
|
110
110
|
# We may run afoul of a synchronous bootstrap process doing the same thing. So
|
111
111
|
# wait until we've managed to run successfully.
|
@@ -457,7 +457,7 @@ end
|
|
457
457
|
end
|
458
458
|
|
459
459
|
if name
|
460
|
-
subnet_mu_name ||= @config['scrub_mu_isms'] ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
|
460
|
+
subnet_mu_name ||= (@config['scrub_mu_isms'] or !@deploy) ? @cloud_id+name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(name, max_length: 61))
|
461
461
|
end
|
462
462
|
|
463
463
|
MU.log "getSubnet(cloud_id: #{cloud_id}, name: #{name}, tag_key: #{tag_key}, tag_value: #{tag_value}, ip_block: #{ip_block}, region: #{region}, subnet_mu_name: #{subnet_mu_name})", MU::DEBUG, details: caller[0]
|
@@ -976,12 +976,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
976
976
|
# @return [String]
|
977
977
|
def getUnusedAddressBlock(exclude: [], max_bits: 28)
|
978
978
|
used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
|
979
|
-
|
980
|
-
used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
|
981
|
-
if s.cloud_desc.secondary_ip_ranges
|
982
|
-
used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
|
983
|
-
end
|
984
|
-
}
|
979
|
+
used_ranges.concat(listSubnetRanges)
|
985
980
|
# XXX sort used_ranges
|
986
981
|
candidate = used_ranges.first.next_sib
|
987
982
|
|
@@ -1003,8 +998,55 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1003
998
|
candidate.to_s
|
1004
999
|
end
|
1005
1000
|
|
1001
|
+
# Add a new secondary IP range to the given subnet, if it doesn't
|
1002
|
+
# already exist
|
1003
|
+
def addSecondaryRange(subnet, cidr, name)
|
1004
|
+
subnet = getSubnet(cloud_id: subnet, name: subnet, subnet_mu_name: subnet)
|
1005
|
+
if !subnet
|
1006
|
+
raise MuError, "#{self.to_s} failed to locate a subnet from '#{subnet}'"
|
1007
|
+
end
|
1008
|
+
|
1009
|
+
secondary_ranges = subnet.cloud_desc.secondary_ip_ranges
|
1010
|
+
secondary_ranges ||= []
|
1011
|
+
secondary_ranges.each { |r|
|
1012
|
+
if r.ip_cidr_range == cidr and r.range_name == name
|
1013
|
+
return
|
1014
|
+
elsif r.ip_cidr_range == cidr or r.range_name == name
|
1015
|
+
MU.log "Conflicting secondary IP range, cannot add #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}", MU::WARN, details: r
|
1016
|
+
return
|
1017
|
+
end
|
1018
|
+
}
|
1019
|
+
|
1020
|
+
secondary_ranges << MU::Cloud::Google.compute(:SubnetworkSecondaryRange).new(
|
1021
|
+
ip_cidr_range: cidr,
|
1022
|
+
range_name: name
|
1023
|
+
)
|
1024
|
+
MU.log "Adding new secondary IP range #{name} (#{cidr}) to network #{cloud_desc.name} subnet #{subnet.cloud_desc.name}"
|
1025
|
+
subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
|
1026
|
+
name: subnet.cloud_desc.name,
|
1027
|
+
secondary_ip_ranges: secondary_ranges,
|
1028
|
+
fingerprint: subnet.cloud_desc.fingerprint
|
1029
|
+
)
|
1030
|
+
MU::Cloud::Google.compute(credentials: @credentials).patch_subnetwork(@project_id, subnet.az, subnet.cloud_desc.name, subnetobj)
|
1031
|
+
end
|
1032
|
+
|
1033
|
+
def connector(id: nil, name: nil)
|
1034
|
+
end
|
1035
|
+
|
1006
1036
|
private
|
1007
1037
|
|
1038
|
+
# @return [Array<NetAddr::IPv4Net>]
|
1039
|
+
def listSubnetRanges
|
1040
|
+
ranges = []
|
1041
|
+
subnets.each { |s|
|
1042
|
+
ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
|
1043
|
+
if s.cloud_desc.secondary_ip_ranges
|
1044
|
+
ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
|
1045
|
+
end
|
1046
|
+
}
|
1047
|
+
ranges
|
1048
|
+
end
|
1049
|
+
|
1008
1050
|
def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
|
1009
1051
|
private_acl = {
|
1010
1052
|
"name" => vpc_name+"-rt",
|
@@ -1209,6 +1251,10 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1209
1251
|
# @return [Boolean]
|
1210
1252
|
def private?
|
1211
1253
|
@parent.cloud_desc
|
1254
|
+
if !@parent.routes
|
1255
|
+
MU.log "Failed to retrieve routes from #{@parent.to_s}", MU::WARN
|
1256
|
+
return true
|
1257
|
+
end
|
1212
1258
|
@parent.routes.map { |r|
|
1213
1259
|
if r.dest_range == "0.0.0.0/0" and !r.next_hop_gateway.nil? and
|
1214
1260
|
(r.tags.nil? or r.tags.size == 0) and
|
@@ -29,8 +29,6 @@ module MU
|
|
29
29
|
@@authorizers = {}
|
30
30
|
@@acct_to_profile_map = {}
|
31
31
|
@@enable_semaphores = {}
|
32
|
-
@@readonly_semaphore = Mutex.new
|
33
|
-
@@readonly = {}
|
34
32
|
|
35
33
|
# Module used by {MU::Cloud} to insert additional instance methods into
|
36
34
|
# instantiated resources in this cloud layer.
|
@@ -40,7 +38,7 @@ module MU
|
|
40
38
|
# @return [String]
|
41
39
|
def url
|
42
40
|
desc = cloud_desc
|
43
|
-
(desc and desc.self_link) ? desc.self_link : nil
|
41
|
+
(desc and desc.respond_to?(:self_link) and desc.self_link) ? desc.self_link : nil
|
44
42
|
end
|
45
43
|
end
|
46
44
|
|
@@ -52,6 +50,12 @@ module MU
|
|
52
50
|
[:url]
|
53
51
|
end
|
54
52
|
|
53
|
+
# Cloud-specific resource methods or attributes we want exposed for
|
54
|
+
# reading by {MU::Cloud}
|
55
|
+
def self.customAttrReaders
|
56
|
+
[:project_id, :customer]
|
57
|
+
end
|
58
|
+
|
55
59
|
# Is this a "real" cloud provider, or a stub like CloudFormation?
|
56
60
|
def self.virtual?
|
57
61
|
false
|
@@ -75,12 +79,6 @@ module MU
|
|
75
79
|
# @param cloudobj [MU::Cloud]
|
76
80
|
# @param deploy [MU::MommaCat]
|
77
81
|
def self.resourceInitHook(cloudobj, deploy)
|
78
|
-
class << self
|
79
|
-
attr_reader :project_id
|
80
|
-
attr_reader :customer
|
81
|
-
# url is too complex for an attribute (we get it from the cloud API),
|
82
|
-
# so it's up in AdditionalResourceMethods instead
|
83
|
-
end
|
84
82
|
return if !cloudobj
|
85
83
|
|
86
84
|
cloudobj.instance_variable_set(:@customer, MU::Cloud::Google.customerID(cloudobj.config['credentials']))
|
@@ -96,9 +94,6 @@ module MU
|
|
96
94
|
cloudobj.instance_variable_set(:@project_id, cloudobj.config['project'])
|
97
95
|
end
|
98
96
|
|
99
|
-
# XXX @url? Well we're not likely to have @cloud_desc at this point, so maybe
|
100
|
-
# that needs to be a generic-to-google wrapper like def url; cloud_desc.self_link;end
|
101
|
-
|
102
97
|
# XXX something like: vpc["habitat"] = MU::Cloud::Google.projectToRef(vpc["project"], config: configurator, credentials: vpc["credentials"])
|
103
98
|
end
|
104
99
|
|
@@ -346,8 +341,10 @@ module MU
|
|
346
341
|
end
|
347
342
|
|
348
343
|
# Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
|
349
|
-
# @param
|
344
|
+
# @param deploy [String]: The deploy for which we're writing the secret
|
350
345
|
# @param value [String]: The contents of the secret
|
346
|
+
# @param name [String]: File/object name
|
347
|
+
# @param credentials [String]
|
351
348
|
def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
|
352
349
|
deploy_id = deploy.deploy_id
|
353
350
|
name ||= deploy_id+"-secret"
|
@@ -847,7 +844,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
847
844
|
require 'google/apis/iam_v1'
|
848
845
|
|
849
846
|
if subclass.nil?
|
850
|
-
@@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials)
|
847
|
+
@@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, retry_readonly: true)
|
851
848
|
return @@iam_api[credentials]
|
852
849
|
elsif subclass.is_a?(Symbol)
|
853
850
|
return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
|
@@ -863,28 +860,12 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
863
860
|
# dopey extra warnings about falling back on scopes
|
864
861
|
credentials ||= MU::Cloud::Google.credConfig(credentials, name_only: true)
|
865
862
|
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
end
|
873
|
-
|
874
|
-
if subclass.nil?
|
875
|
-
begin
|
876
|
-
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: use_scopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true)
|
877
|
-
rescue Signet::AuthorizationError
|
878
|
-
MU.log "Falling back to read-only access to DirectoryService API for credential set '#{credentials}'", MU::WARN
|
879
|
-
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: readscopes, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
|
880
|
-
@@readonly[credentials] ||= {}
|
881
|
-
@@readonly[credentials]["AdminDirectoryV1"] = true
|
882
|
-
end
|
883
|
-
return @@admin_directory_api[credentials]
|
884
|
-
elsif subclass.is_a?(Symbol)
|
885
|
-
return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
|
886
|
-
end
|
887
|
-
}
|
863
|
+
if subclass.nil?
|
864
|
+
@@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: ['admin.directory.group.member', 'admin.directory.group', 'admin.directory.user', 'admin.directory.domain', 'admin.directory.orgunit', 'admin.directory.rolemanagement', 'admin.directory.customer', 'admin.directory.user.alias', 'admin.directory.userschema'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials, auth_error_quiet: true, retry_readonly: true)
|
865
|
+
return @@admin_directory_api[credentials]
|
866
|
+
elsif subclass.is_a?(Symbol)
|
867
|
+
return Object.const_get("::Google").const_get("Apis").const_get("AdminDirectoryV1").const_get(subclass)
|
868
|
+
end
|
888
869
|
end
|
889
870
|
|
890
871
|
# Google's Cloud Resource Manager API
|
@@ -896,7 +877,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
896
877
|
if !MU::Cloud::Google.credConfig(credentials)
|
897
878
|
raise MuError, "No such credential set #{credentials} defined in mu.yaml!"
|
898
879
|
end
|
899
|
-
@@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'])
|
880
|
+
@@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['cloud-platform', 'cloudplatformprojects', 'cloudplatformorganizations', 'cloudplatformfolders'], credentials: credentials, masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], retry_readonly: true)
|
900
881
|
return @@resource_api[credentials]
|
901
882
|
elsif subclass.is_a?(Symbol)
|
902
883
|
return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
|
@@ -1110,7 +1091,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1110
1091
|
# Create a Google Cloud Platform API client
|
1111
1092
|
# @param api [String]: Which API are we wrapping?
|
1112
1093
|
# @param scopes [Array<String>]: Google auth scopes applicable to this API
|
1113
|
-
def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
|
1094
|
+
def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false, retry_readonly: false)
|
1114
1095
|
@credentials = credentials
|
1115
1096
|
@scopes = scopes.map { |s|
|
1116
1097
|
if !s.match(/\//) # allow callers to use shorthand
|
@@ -1127,6 +1108,22 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1127
1108
|
@api.authorization.sub = @masquerade
|
1128
1109
|
@api.authorization.fetch_access_token!
|
1129
1110
|
rescue Signet::AuthorizationError => e
|
1111
|
+
if retry_readonly
|
1112
|
+
newscopes = @scopes.map { |s|
|
1113
|
+
if s =~ /\/cloud-platform$/
|
1114
|
+
s += ".read-only"
|
1115
|
+
elsif s !~ /\/cloud-platform\b/ and s !~ /\.readonly$/
|
1116
|
+
s += ".readonly"
|
1117
|
+
end
|
1118
|
+
s
|
1119
|
+
}
|
1120
|
+
if newscopes != @scopes
|
1121
|
+
MU.log "Falling back to read-only access to #{api} API on credential set '#{credentials}'", MU::WARN, details: @scopes
|
1122
|
+
@scopes = newscopes
|
1123
|
+
@api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials)
|
1124
|
+
retry
|
1125
|
+
end
|
1126
|
+
end
|
1130
1127
|
if auth_error_quiet
|
1131
1128
|
MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
|
1132
1129
|
else
|
@@ -1281,7 +1278,7 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1281
1278
|
raise e
|
1282
1279
|
end
|
1283
1280
|
rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
|
1284
|
-
if e.message.match(/^quotaExceeded: Request rate/)
|
1281
|
+
if e.message.match(/^quotaExceeded: Request rate|failedPrecondition.*?already in progress/)
|
1285
1282
|
if retries <= 10
|
1286
1283
|
sleep wait_backoff
|
1287
1284
|
retries += 1
|
@@ -1457,6 +1454,9 @@ MU.log e.message, MU::WARN, details: e.inspect
|
|
1457
1454
|
faked_args.pop
|
1458
1455
|
faked_args.pop
|
1459
1456
|
end
|
1457
|
+
if method_sym == :patch_subnetwork
|
1458
|
+
faked_args.pop
|
1459
|
+
end
|
1460
1460
|
faked_args.push(cloud_id)
|
1461
1461
|
if get_method == :get_project_location_cluster
|
1462
1462
|
faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
|
data/modules/mu.rb
CHANGED
@@ -701,7 +701,7 @@ module MU
|
|
701
701
|
!$MU_CFG['public_address'].empty? and @@my_public_ip != $MU_CFG['public_address']
|
702
702
|
@@mu_public_addr = $MU_CFG['public_address']
|
703
703
|
if !@@mu_public_addr.match(/^\d+\.\d+\.\d+\.\d+$/) and
|
704
|
-
File.
|
704
|
+
File.exist?("/etc/hostname") and File.exist?("/etc/hosts")
|
705
705
|
hostname = IO.readlines("/etc/hostname")[0].gsub(/\n/, '')
|
706
706
|
|
707
707
|
hostlines = File.open('/etc/hosts').grep(/.*#{hostname}.*/)
|
@@ -788,7 +788,7 @@ module MU
|
|
788
788
|
end
|
789
789
|
|
790
790
|
# The version of Chef we will install on nodes.
|
791
|
-
@@chefVersion = "
|
791
|
+
@@chefVersion = "18.5.0"
|
792
792
|
# The version of Chef we will install on nodes.
|
793
793
|
# @return [String]
|
794
794
|
def self.chefVersion
|
@@ -1072,23 +1072,23 @@ module MU
|
|
1072
1072
|
end
|
1073
1073
|
|
1074
1074
|
|
1075
|
-
# Generate a random password which will satisfy the complexity requirements of stock Amazon Windows AMIs.
|
1075
|
+
# Generate a random password which will satisfy the complexity requirements of things like the stock Amazon Windows AMIs.
|
1076
1076
|
# return [String]: A password string.
|
1077
|
-
def self.
|
1077
|
+
def self.generatePassword(safe_pattern: '~!@#%^&*_-+=`|(){}[]:;<>,.?', length: 14, retries: 50)
|
1078
1078
|
# We have dopey complexity requirements, be stringent here.
|
1079
1079
|
# I'll be nice and not condense this into one elegant-but-unreadable regular expression
|
1080
1080
|
attempts = 0
|
1081
1081
|
safe_metachars = Regexp.escape(safe_pattern)
|
1082
1082
|
begin
|
1083
1083
|
if attempts > retries
|
1084
|
-
MU.log "Failed to generate an adequate
|
1085
|
-
raise MuError, "Failed to generate an adequate
|
1084
|
+
MU.log "Failed to generate an adequate password after #{attempts} attempts", MU::ERR
|
1085
|
+
raise MuError, "Failed to generate an adequate password after #{attempts} attempts"
|
1086
1086
|
end
|
1087
|
-
winpass = Password.random(
|
1087
|
+
winpass = Password.random(length..(length+2))
|
1088
1088
|
attempts += 1
|
1089
1089
|
end while winpass.nil? or !winpass.match(/^[a-z]/i) or !winpass.match(/[A-Z]/) or !winpass.match(/[a-z]/) or !winpass.match(/\d/) or !winpass.match(/[#{safe_metachars}]/) or winpass.match(/[^\w\d#{safe_metachars}]/)
|
1090
1090
|
|
1091
|
-
MU.log "Generated
|
1091
|
+
MU.log "Generated password after #{attempts} attempts", MU::DEBUG
|
1092
1092
|
return winpass
|
1093
1093
|
end
|
1094
1094
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# groomers: Ansible
|
2
|
+
---
|
3
|
+
appname: smoketest
|
4
|
+
generate_passwords:
|
5
|
+
- itemname: elasticpw
|
6
|
+
minlength: 12
|
7
|
+
vpcs:
|
8
|
+
- name: wrapper
|
9
|
+
servers:
|
10
|
+
- name: frontend
|
11
|
+
platform: centos7
|
12
|
+
groomer: Ansible
|
13
|
+
vpc:
|
14
|
+
name: wrapper
|
15
|
+
size: t3.medium
|
16
|
+
vault_access:
|
17
|
+
- item: elasticpw
|
18
|
+
run_list:
|
19
|
+
- mu-logstash
|
20
|
+
#<% if cloud != "AWS" %>
|
21
|
+
- name: backend
|
22
|
+
platform: centos7
|
23
|
+
groomer: Ansible
|
24
|
+
vpc:
|
25
|
+
name: wrapper
|
26
|
+
size: m5.large
|
27
|
+
vault_access:
|
28
|
+
- item: elasticpw
|
29
|
+
run_list:
|
30
|
+
- mu-elastic
|
31
|
+
#<% else %>
|
32
|
+
#search_domains:
|
33
|
+
#- name: logsearch
|
34
|
+
# elasticsearch_version: '7.4'
|
35
|
+
# instance_count: 1
|
36
|
+
# instance_type: r5.large.elasticsearch
|
37
|
+
# ebs_size: 10
|
38
|
+
# ebs_type: gp2
|
39
|
+
# access_policies:
|
40
|
+
# Version: '2012-10-17'
|
41
|
+
# Statement:
|
42
|
+
# - Effect: Allow
|
43
|
+
# Principal:
|
44
|
+
# AWS: "*"
|
45
|
+
# Action: es:ESHttp*
|
46
|
+
#<% end %>
|
@@ -15,7 +15,7 @@ control 'init' do
|
|
15
15
|
node = json('/tmp/chef_node.json').params
|
16
16
|
NODE_PUB_IP=node_meta[0]['pub_ip']
|
17
17
|
CHEF_SERVER_VERSION="12.17.15-1"
|
18
|
-
CHEF_CLIENT_VERSION="
|
18
|
+
CHEF_CLIENT_VERSION="18.2.7"
|
19
19
|
KNIFE_WINDOWS="1.9.0"
|
20
20
|
MU_BASE="/opt/mu"
|
21
21
|
f = "/etc/ssh/sshd_config"
|