cloud-mu 3.1.4 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +5 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +158 -107
- data/modules/mu/adoption.rb +386 -59
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +135 -82
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +123 -21
- data/modules/mu/groomers/chef.rb +64 -11
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/master/ssl.rb +0 -1
- data/modules/mu/mommacat.rb +154 -867
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +225 -192
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +281 -64
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
- data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +14 -8
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +142 -55
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +25 -0
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +169 -93
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
- data/modules/tests/needwork/win2k12.yaml +0 -13
|
@@ -120,7 +120,7 @@ module MU
|
|
|
120
120
|
if !@deploy.nocleanup
|
|
121
121
|
Thread.new {
|
|
122
122
|
MU.dupGlobals(parent_thread_id)
|
|
123
|
-
MU::Cloud
|
|
123
|
+
MU::Cloud.resourceClass("AWS", "Server").terminateInstance(id: member.instance_id)
|
|
124
124
|
}
|
|
125
125
|
end
|
|
126
126
|
end
|
|
@@ -193,9 +193,10 @@ module MU
|
|
|
193
193
|
# @return [Array<MU::Cloud::Server>]
|
|
194
194
|
def listNodes
|
|
195
195
|
nodes = []
|
|
196
|
-
me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id)
|
|
197
|
-
|
|
198
|
-
|
|
196
|
+
me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first
|
|
197
|
+
pp me
|
|
198
|
+
if me and me.instances
|
|
199
|
+
me.instances.each { |instance|
|
|
199
200
|
found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true)
|
|
200
201
|
nodes.concat(found)
|
|
201
202
|
}
|
|
@@ -425,6 +426,7 @@ module MU
|
|
|
425
426
|
# @return [OpenStruct]
|
|
426
427
|
def cloud_desc(use_cache: true)
|
|
427
428
|
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
|
429
|
+
return nil if !@cloud_id
|
|
428
430
|
@cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
|
|
429
431
|
auto_scaling_group_names: [@mu_name]
|
|
430
432
|
).auto_scaling_groups.first
|
|
@@ -531,14 +533,25 @@ module MU
|
|
|
531
533
|
if cloud_desc.vpc_zone_identifier and
|
|
532
534
|
!cloud_desc.vpc_zone_identifier.empty?
|
|
533
535
|
nets = cloud_desc.vpc_zone_identifier.split(/,/)
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
536
|
+
begin
|
|
537
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
|
|
538
|
+
bok['vpc'] = MU::Config::Ref.get(
|
|
539
|
+
id: resp.vpc_id,
|
|
540
|
+
cloud: "AWS",
|
|
541
|
+
credentials: @credentials,
|
|
542
|
+
type: "vpcs",
|
|
543
|
+
subnets: nets.map { |s| { "subnet_id" => s } }
|
|
544
|
+
)
|
|
545
|
+
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
|
546
|
+
if e.message.match(/The subnet ID '(subnet-[a-f0-9]+)' does not exist/)
|
|
547
|
+
nets.delete(Regexp.last_match[1])
|
|
548
|
+
if nets.empty?
|
|
549
|
+
MU.log "Autoscale Group #{@cloud_id} was configured for a VPC, but the configuration held no valid subnets", MU::WARN, details: cloud_desc.vpc_zone_identifier.split(/,/)
|
|
550
|
+
end
|
|
551
|
+
else
|
|
552
|
+
raise e
|
|
553
|
+
end
|
|
554
|
+
end
|
|
542
555
|
end
|
|
543
556
|
|
|
544
557
|
# MU.log @cloud_id, MU::NOTICE, details: cloud_desc
|
|
@@ -813,26 +826,7 @@ module MU
|
|
|
813
826
|
}
|
|
814
827
|
}
|
|
815
828
|
},
|
|
816
|
-
"ingress_rules" =>
|
|
817
|
-
"items" => {
|
|
818
|
-
"properties" => {
|
|
819
|
-
"sgs" => {
|
|
820
|
-
"type" => "array",
|
|
821
|
-
"items" => {
|
|
822
|
-
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
|
823
|
-
"type" => "string"
|
|
824
|
-
}
|
|
825
|
-
},
|
|
826
|
-
"lbs" => {
|
|
827
|
-
"type" => "array",
|
|
828
|
-
"items" => {
|
|
829
|
-
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
|
830
|
-
"type" => "string"
|
|
831
|
-
}
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
}
|
|
835
|
-
}
|
|
829
|
+
"ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
|
|
836
830
|
}
|
|
837
831
|
[toplevel_required, schema]
|
|
838
832
|
end
|
|
@@ -905,7 +899,7 @@ module MU
|
|
|
905
899
|
launch = pool["basis"]["launch_config"]
|
|
906
900
|
launch['iam_policies'] ||= pool['iam_policies']
|
|
907
901
|
|
|
908
|
-
launch['size'] = MU::Cloud
|
|
902
|
+
launch['size'] = MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(launch["size"], pool["region"])
|
|
909
903
|
ok = false if launch['size'].nil?
|
|
910
904
|
if !launch['generate_iam_role']
|
|
911
905
|
if !launch['iam_role'] and pool['cloud'] != "CloudFormation"
|
|
@@ -949,11 +943,7 @@ module MU
|
|
|
949
943
|
|
|
950
944
|
role['credentials'] = pool['credentials'] if pool['credentials']
|
|
951
945
|
configurator.insertKitten(role, "roles")
|
|
952
|
-
pool["
|
|
953
|
-
pool["dependencies"] << {
|
|
954
|
-
"type" => "role",
|
|
955
|
-
"name" => pool["name"]
|
|
956
|
-
}
|
|
946
|
+
MU::Config.addDependency(pool, pool['name'], "role")
|
|
957
947
|
end
|
|
958
948
|
launch["ami_id"] ||= launch["image_id"]
|
|
959
949
|
if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
|
|
@@ -967,7 +957,7 @@ module MU
|
|
|
967
957
|
end
|
|
968
958
|
end
|
|
969
959
|
if launch["server"] != nil
|
|
970
|
-
pool["
|
|
960
|
+
MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
|
|
971
961
|
# XXX I dunno, maybe toss an error if this isn't done already
|
|
972
962
|
# servers.each { |server|
|
|
973
963
|
# if server["name"] == launch["server"]
|
|
@@ -1073,7 +1063,7 @@ module MU
|
|
|
1073
1063
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
1074
1064
|
# @param region [String]: The cloud provider region
|
|
1075
1065
|
# @return [void]
|
|
1076
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
1066
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
1077
1067
|
MU.log "AWS::ServerPool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
1078
1068
|
|
|
1079
1069
|
filters = [{name: "key", values: ["MU-ID"]}]
|
|
@@ -1096,7 +1086,7 @@ module MU
|
|
|
1096
1086
|
if asg.key == "MU-MASTER-IP" and asg.value != MU.mu_public_ip and !ignoremaster
|
|
1097
1087
|
no_purge << asg.resource_id
|
|
1098
1088
|
end
|
|
1099
|
-
if asg.key == "MU-ID" and asg.value ==
|
|
1089
|
+
if asg.key == "MU-ID" and asg.value == deploy_id
|
|
1100
1090
|
maybe_purge << asg.resource_id
|
|
1101
1091
|
end
|
|
1102
1092
|
}
|
|
@@ -1123,7 +1113,7 @@ module MU
|
|
|
1123
1113
|
end
|
|
1124
1114
|
end
|
|
1125
1115
|
|
|
1126
|
-
# MU::Cloud
|
|
1116
|
+
# MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(resource_id)
|
|
1127
1117
|
|
|
1128
1118
|
# Generally there should be a launch_configuration of the same name
|
|
1129
1119
|
# XXX search for these independently, too?
|
|
@@ -1164,14 +1154,14 @@ module MU
|
|
|
1164
1154
|
@config['basis']['launch_config']["ami_id"] = @deploy.deployment["images"][@config['basis']['launch_config']["server"]]["image_id"]
|
|
1165
1155
|
MU.log "Using AMI '#{@config['basis']['launch_config']["ami_id"]}' from sibling server #{@config['basis']['launch_config']["server"]} in ServerPool #{@mu_name}"
|
|
1166
1156
|
elsif !@config['basis']['launch_config']["instance_id"].nil?
|
|
1167
|
-
@config['basis']['launch_config']["ami_id"] = MU::Cloud
|
|
1157
|
+
@config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
|
|
1168
1158
|
name: @mu_name,
|
|
1169
1159
|
instance_id: @config['basis']['launch_config']["instance_id"],
|
|
1170
1160
|
credentials: @config['credentials'],
|
|
1171
1161
|
region: @config['region']
|
|
1172
1162
|
)[@config['region']]
|
|
1173
1163
|
end
|
|
1174
|
-
MU::Cloud
|
|
1164
|
+
MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
|
|
1175
1165
|
|
|
1176
1166
|
oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
|
|
1177
1167
|
launch_configuration_names: [@mu_name]
|
|
@@ -1226,12 +1216,12 @@ module MU
|
|
|
1226
1216
|
vol.delete("encrypted")
|
|
1227
1217
|
end
|
|
1228
1218
|
end
|
|
1229
|
-
mapping, _cfm_mapping = MU::Cloud
|
|
1219
|
+
mapping, _cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
|
|
1230
1220
|
storage << mapping
|
|
1231
1221
|
}
|
|
1232
1222
|
end
|
|
1233
1223
|
|
|
1234
|
-
storage.concat(MU::Cloud
|
|
1224
|
+
storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
|
|
1235
1225
|
|
|
1236
1226
|
if @config['basis']['launch_config']['generate_iam_role']
|
|
1237
1227
|
role = @deploy.findLitterMate(name: @config['name'], type: "roles")
|
|
@@ -67,7 +67,7 @@ module MU
|
|
|
67
67
|
if target['vpc']["subnet_name"]
|
|
68
68
|
subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
|
|
69
69
|
if subnet_obj.nil?
|
|
70
|
-
raise MuError, "Failed to locate subnet from #{
|
|
70
|
+
raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
|
|
71
71
|
end
|
|
72
72
|
target['vpc']['subnet_id'] = subnet_obj.cloud_id
|
|
73
73
|
end
|
|
@@ -261,49 +261,29 @@ module MU
|
|
|
261
261
|
targets = {}
|
|
262
262
|
|
|
263
263
|
if @config['mount_points'] && !@config['mount_points'].empty?
|
|
264
|
+
mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
|
265
|
+
file_system_id: storage_pool.file_system_id
|
|
266
|
+
).mount_targets
|
|
267
|
+
|
|
264
268
|
@config['mount_points'].each { |mp|
|
|
265
269
|
subnet = nil
|
|
266
270
|
dependencies
|
|
267
|
-
mp_vpc =
|
|
268
|
-
@deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
|
|
269
|
-
elsif mp['vpc']
|
|
270
|
-
MU::MommaCat.findStray(
|
|
271
|
-
@config['cloud'],
|
|
272
|
-
"vpcs",
|
|
273
|
-
deploy_id: mp['vpc']["deploy_id"],
|
|
274
|
-
credentials: @config['credentials'],
|
|
275
|
-
mu_name: mp['vpc']["mu_name"],
|
|
276
|
-
cloud_id: mp['vpc']['vpc_id'],
|
|
277
|
-
region: @config['region'],
|
|
278
|
-
dummy_ok: false
|
|
279
|
-
).first
|
|
280
|
-
# XXX non-sibling, findStray version
|
|
281
|
-
end
|
|
271
|
+
mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten
|
|
282
272
|
|
|
283
|
-
mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
|
284
|
-
file_system_id: storage_pool.file_system_id
|
|
285
|
-
).mount_targets
|
|
286
273
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
274
|
+
subnet_obj = mp_vpc.subnets.select { |s|
|
|
275
|
+
s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
|
|
276
|
+
}.first
|
|
290
277
|
mount_target = nil
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
}
|
|
299
|
-
break if mount_target
|
|
278
|
+
mount_targets.each { |t|
|
|
279
|
+
subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
|
|
280
|
+
if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
|
|
281
|
+
mount_target = t
|
|
282
|
+
subnet = subnet_obj.cloud_desc
|
|
283
|
+
break
|
|
284
|
+
end
|
|
300
285
|
}
|
|
301
286
|
|
|
302
|
-
# mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
|
303
|
-
# mount_target_id: mp["cloud_id"]
|
|
304
|
-
# ).mount_targets.first
|
|
305
|
-
|
|
306
|
-
|
|
307
287
|
targets[mp["name"]] = {
|
|
308
288
|
"owner_id" => mount_target.owner_id,
|
|
309
289
|
"cloud_id" => mount_target.mount_target_id,
|
|
@@ -353,7 +333,7 @@ module MU
|
|
|
353
333
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
354
334
|
# @param region [String]: The cloud provider region in which to operate
|
|
355
335
|
# @return [void]
|
|
356
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
336
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
357
337
|
MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
358
338
|
|
|
359
339
|
supported_regions = %w{us-west-2 us-east-1 eu-west-1}
|
|
@@ -378,7 +358,7 @@ module MU
|
|
|
378
358
|
found_muid = false
|
|
379
359
|
found_master = false
|
|
380
360
|
tags.each { |tag|
|
|
381
|
-
found_muid = true if tag.key == "MU-ID" && tag.value ==
|
|
361
|
+
found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
|
|
382
362
|
found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
|
|
383
363
|
}
|
|
384
364
|
next if !found_muid
|
|
@@ -493,6 +473,9 @@ module MU
|
|
|
493
473
|
|
|
494
474
|
if pool['mount_points'] && !pool['mount_points'].empty?
|
|
495
475
|
pool['mount_points'].each{ |mp|
|
|
476
|
+
if mp['vpc'] and mp['vpc']['name']
|
|
477
|
+
MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
|
|
478
|
+
end
|
|
496
479
|
if mp['ingress_rules']
|
|
497
480
|
fwname = "storage-#{mp['name']}"
|
|
498
481
|
acl = {
|
|
@@ -109,7 +109,7 @@ module MU
|
|
|
109
109
|
# Create these if necessary, then append them to the list of
|
|
110
110
|
# attachable_policies
|
|
111
111
|
if @config['raw_policies']
|
|
112
|
-
pol_arns = MU::Cloud
|
|
112
|
+
pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
|
|
113
113
|
@config['raw_policies'],
|
|
114
114
|
basename: @deploy.getResourceName(@config['name']),
|
|
115
115
|
credentials: @credentials
|
|
@@ -135,7 +135,7 @@ module MU
|
|
|
135
135
|
attached_policies.each { |a|
|
|
136
136
|
if !configured_policies.include?(a.policy_arn)
|
|
137
137
|
MU.log "Removing IAM policy #{a.policy_arn} from user #{@mu_name}", MU::NOTICE
|
|
138
|
-
MU::Cloud
|
|
138
|
+
MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
|
|
139
139
|
else
|
|
140
140
|
configured_policies.delete(a.policy_arn)
|
|
141
141
|
end
|
|
@@ -151,7 +151,7 @@ module MU
|
|
|
151
151
|
end
|
|
152
152
|
|
|
153
153
|
if @config['inline_policies']
|
|
154
|
-
docs = MU::Cloud
|
|
154
|
+
docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
|
|
155
155
|
docs.each { |doc|
|
|
156
156
|
MU.log "Putting user policy #{doc.keys.first} to user #{@cloud_id} "
|
|
157
157
|
MU::Cloud::AWS.iam(credentials: @credentials).put_user_policy(
|
|
@@ -190,16 +190,16 @@ module MU
|
|
|
190
190
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
191
191
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
192
192
|
# @return [void]
|
|
193
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
193
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
194
194
|
MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
195
195
|
|
|
196
196
|
# XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
|
|
197
197
|
resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
|
|
198
|
-
path_prefix: "/"+
|
|
198
|
+
path_prefix: "/"+deploy_id+"/"
|
|
199
199
|
)
|
|
200
200
|
if resp and resp.policies
|
|
201
201
|
resp.policies.each { |policy|
|
|
202
|
-
MU.log "Deleting policy /#{
|
|
202
|
+
MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}"
|
|
203
203
|
if !noop
|
|
204
204
|
attachments = begin
|
|
205
205
|
MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
|
|
@@ -277,7 +277,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
|
277
277
|
has_ourdeploy = false
|
|
278
278
|
has_ourmaster = false
|
|
279
279
|
tags.each { |tag|
|
|
280
|
-
if tag.key == "MU-ID" and tag.value ==
|
|
280
|
+
if tag.key == "MU-ID" and tag.value == deploy_id
|
|
281
281
|
has_ourdeploy = true
|
|
282
282
|
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
|
283
283
|
has_ourmaster = true
|
|
@@ -431,7 +431,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
|
431
431
|
resp.policy_names.each { |pol_name|
|
|
432
432
|
pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
|
|
433
433
|
doc = JSON.parse(URI.decode(pol.policy_document))
|
|
434
|
-
bok["inline_policies"] = MU::Cloud
|
|
434
|
+
bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
|
|
435
435
|
}
|
|
436
436
|
end
|
|
437
437
|
|
|
@@ -465,7 +465,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
|
465
465
|
def self.schema(_config)
|
|
466
466
|
toplevel_required = []
|
|
467
467
|
polschema = MU::Config::Role.schema["properties"]["policies"]
|
|
468
|
-
polschema.deep_merge!(MU::Cloud
|
|
468
|
+
polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
|
|
469
469
|
|
|
470
470
|
schema = {
|
|
471
471
|
"inline_policies" => polschema,
|
|
@@ -517,7 +517,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
|
517
517
|
# If we're attaching some managed policies, make sure all of the ones
|
|
518
518
|
# that should already exist do indeed exist
|
|
519
519
|
if user['attachable_policies']
|
|
520
|
-
ok = false if !MU::Cloud
|
|
520
|
+
ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
|
|
521
521
|
user['attachable_policies'],
|
|
522
522
|
credentials: user['credentials'],
|
|
523
523
|
region: user['region']
|
|
@@ -530,7 +530,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
|
530
530
|
if configurator.haveLitterMate?(group, "groups")
|
|
531
531
|
need_dependency = true
|
|
532
532
|
else
|
|
533
|
-
found = MU::Cloud
|
|
533
|
+
found = MU::Cloud.resourceClass("AWS", "Group").find(cloud_id: group)
|
|
534
534
|
if found.nil? or found.empty? or (configurator.updating and
|
|
535
535
|
found.values.first.group.path == "/"+configurator.updating+"/")
|
|
536
536
|
groupdesc = {
|
|
@@ -542,11 +542,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
|
542
542
|
end
|
|
543
543
|
|
|
544
544
|
if need_dependency
|
|
545
|
-
user
|
|
546
|
-
user["dependencies"] << {
|
|
547
|
-
"type" => "group",
|
|
548
|
-
"name" => group
|
|
549
|
-
}
|
|
545
|
+
MU::Config.addDependency(user, group, "group")
|
|
550
546
|
end
|
|
551
547
|
}
|
|
552
548
|
end
|
|
File without changes
|
|
@@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
|
42
42
|
<% if !$mu.skipApplyUpdates %>
|
|
43
43
|
set +e
|
|
44
44
|
if [ ! -f /.mu-installer-ran-updates ];then
|
|
45
|
-
|
|
45
|
+
echo "Applying package updates" > /etc/nologin
|
|
46
46
|
apt-get --fix-missing -y upgrade
|
|
47
47
|
touch /.mu-installer-ran-updates
|
|
48
48
|
if [ $? -eq 0 ]
|
|
@@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
|
58
58
|
else
|
|
59
59
|
echo "FAILED PACKAGE UPDATE" >&2
|
|
60
60
|
fi
|
|
61
|
-
|
|
61
|
+
rm -f /etc/nologin
|
|
62
62
|
fi
|
|
63
63
|
<% end %>
|
|
64
64
|
elif [ -x /usr/bin/yum ];then
|
|
@@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
|
94
94
|
<% if !$mu.skipApplyUpdates %>
|
|
95
95
|
set +e
|
|
96
96
|
if [ ! -f /.mu-installer-ran-updates ];then
|
|
97
|
-
|
|
97
|
+
echo "Applying package updates" > /etc/nologin
|
|
98
98
|
kernel_update=`yum list updates | grep kernel`
|
|
99
99
|
yum -y update
|
|
100
100
|
touch /.mu-installer-ran-updates
|
|
@@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
|
108
108
|
else
|
|
109
109
|
echo "FAILED PACKAGE UPDATE" >&2
|
|
110
110
|
fi
|
|
111
|
-
|
|
111
|
+
rm -f /etc/nologin
|
|
112
112
|
fi
|
|
113
113
|
<% end %>
|
|
114
114
|
fi
|
|
@@ -116,6 +116,7 @@ else
|
|
|
116
116
|
/bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
|
|
117
117
|
touch /.mu-installer-ran-updates
|
|
118
118
|
fi
|
|
119
|
+
rm -f /etc/nologin
|
|
119
120
|
|
|
120
121
|
AWSCLI='command -v aws'
|
|
121
122
|
PIP='command -v pip'
|
|
@@ -23,7 +23,7 @@ function log
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
function fetchSecret([string]$file){
|
|
26
|
-
log "
|
|
26
|
+
log "aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file"
|
|
27
27
|
aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file
|
|
28
28
|
}
|
|
29
29
|
|
|
@@ -245,6 +245,7 @@ if($ingroup -ne $admin_username){
|
|
|
245
245
|
net localgroup WinRMRemoteWMIUsers__ /add $admin_username
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
+
importCert "$myname-winrm.crt" "root"
|
|
248
249
|
$winrmcert = importCert "$myname-winrm.crt" "TrustedPeople"
|
|
249
250
|
Set-Item -Path WSMan:\localhost\Service\Auth\Certificate -Value $true
|
|
250
251
|
Set-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" -Name LocalAccountTokenFilterPolicy -Value 1
|
|
@@ -18,6 +18,7 @@ module MU
|
|
|
18
18
|
|
|
19
19
|
# Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
|
|
20
20
|
class VPC < MU::Cloud::VPC
|
|
21
|
+
require 'mu/providers/aws/vpc_subnet'
|
|
21
22
|
|
|
22
23
|
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
|
|
23
24
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
|
@@ -35,78 +36,40 @@ module MU
|
|
|
35
36
|
def create
|
|
36
37
|
MU.log "Creating VPC #{@mu_name}", details: @config
|
|
37
38
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc(cidr_block: @config['ip_block']).vpc
|
|
38
|
-
|
|
39
|
+
@cloud_id = resp.vpc_id
|
|
40
|
+
@config['vpc_id'] = @cloud_id
|
|
39
41
|
|
|
40
|
-
|
|
41
|
-
MU::Cloud::AWS.createTag(vpc_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
|
42
|
-
|
|
43
|
-
if @config['tags']
|
|
44
|
-
@config['tags'].each { |tag|
|
|
45
|
-
MU::Cloud::AWS.createTag(vpc_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
46
|
-
}
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
if @config['optional_tags']
|
|
50
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
51
|
-
MU::Cloud::AWS.createTag(vpc_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
52
|
-
}
|
|
53
|
-
end
|
|
42
|
+
tag_me
|
|
54
43
|
|
|
55
44
|
if resp.state != "available"
|
|
56
45
|
begin
|
|
57
|
-
MU.log "Waiting for VPC #{@mu_name} (#{
|
|
46
|
+
MU.log "Waiting for VPC #{@mu_name} (#{@cloud_id}) to be available", MU::NOTICE
|
|
58
47
|
sleep 5
|
|
59
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [
|
|
48
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [@cloud_id]).vpcs.first
|
|
60
49
|
end while resp.state != "available"
|
|
61
50
|
# There's a default route table that comes with. Let's tag it.
|
|
62
51
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
63
52
|
filters: [
|
|
64
53
|
{
|
|
65
54
|
name: "vpc-id",
|
|
66
|
-
values: [
|
|
55
|
+
values: [@cloud_id]
|
|
67
56
|
}
|
|
68
57
|
]
|
|
69
58
|
)
|
|
70
59
|
resp.route_tables.each { |rtb|
|
|
71
|
-
|
|
72
|
-
if @config['tags']
|
|
73
|
-
@config['tags'].each { |tag|
|
|
74
|
-
MU::Cloud::AWS.createTag(rtb.route_table_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
75
|
-
}
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
MU::Cloud::AWS.createStandardTags(rtb.route_table_id, region: @config['region'], credentials: @config['credentials'])
|
|
79
|
-
|
|
80
|
-
if @config['optional_tags']
|
|
81
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
82
|
-
MU::Cloud::AWS.createTag(rtb.route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
83
|
-
}
|
|
84
|
-
end
|
|
60
|
+
tag_me(rtb.route_table_id, @mu_name+"-#DEFAULTPRIV")
|
|
85
61
|
}
|
|
86
62
|
end
|
|
87
|
-
@config['vpc_id'] = vpc_id
|
|
88
|
-
@cloud_id = vpc_id
|
|
89
63
|
|
|
90
64
|
if @config['create_internet_gateway']
|
|
91
65
|
MU.log "Creating Internet Gateway #{@mu_name}"
|
|
92
66
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_internet_gateway
|
|
93
67
|
internet_gateway_id = resp.internet_gateway.internet_gateway_id
|
|
94
68
|
sleep 5
|
|
95
|
-
MU::Cloud::AWS.createStandardTags(internet_gateway_id, region: @config['region'], credentials: @config['credentials'])
|
|
96
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
|
97
|
-
if @config['tags']
|
|
98
|
-
@config['tags'].each { |tag|
|
|
99
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
100
|
-
}
|
|
101
|
-
end
|
|
102
69
|
|
|
103
|
-
|
|
104
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
105
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
106
|
-
}
|
|
107
|
-
end
|
|
70
|
+
tag_me(internet_gateway_id)
|
|
108
71
|
|
|
109
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id:
|
|
72
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: @cloud_id, internet_gateway_id: internet_gateway_id)
|
|
110
73
|
@config['internet_gateway_id'] = internet_gateway_id
|
|
111
74
|
end
|
|
112
75
|
|
|
@@ -165,236 +128,35 @@ module MU
|
|
|
165
128
|
)
|
|
166
129
|
end
|
|
167
130
|
|
|
168
|
-
nat_gateways =
|
|
169
|
-
if !@config['subnets'].nil?
|
|
170
|
-
allocation_ids = []
|
|
171
|
-
subnetthreads = Array.new
|
|
172
|
-
parent_thread_id = Thread.current.object_id
|
|
173
|
-
azs = []
|
|
174
|
-
@config['subnets'].each { |subnet|
|
|
175
|
-
subnet_name = @config['name']+"-"+subnet['name']
|
|
176
|
-
MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']})", details: subnet
|
|
177
|
-
azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials']) if azs.size == 0
|
|
178
|
-
if !subnet['availability_zone'].nil?
|
|
179
|
-
az = subnet['availability_zone']
|
|
180
|
-
else
|
|
181
|
-
az = azs.pop
|
|
182
|
-
end
|
|
183
|
-
|
|
184
|
-
subnetthreads << Thread.new {
|
|
185
|
-
MU.dupGlobals(parent_thread_id)
|
|
186
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
|
|
187
|
-
vpc_id: vpc_id,
|
|
188
|
-
cidr_block: subnet['ip_block'],
|
|
189
|
-
availability_zone: az
|
|
190
|
-
).subnet
|
|
191
|
-
subnet_id = subnet['subnet_id'] = resp.subnet_id
|
|
192
|
-
MU::Cloud::AWS.createStandardTags(subnet_id, region: @config['region'], credentials: @config['credentials'])
|
|
193
|
-
MU::Cloud::AWS.createTag(subnet_id, "Name", @mu_name+"-"+subnet['name'], region: @config['region'], credentials: @config['credentials'])
|
|
194
|
-
if @config['tags']
|
|
195
|
-
@config['tags'].each { |tag|
|
|
196
|
-
MU::Cloud::AWS.createTag(subnet_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
197
|
-
}
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
if @config['optional_tags']
|
|
201
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
202
|
-
MU::Cloud::AWS.createTag(subnet_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
203
|
-
}
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
retries = 0
|
|
207
|
-
begin
|
|
208
|
-
if resp.state != "available"
|
|
209
|
-
begin
|
|
210
|
-
MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE if retries > 0 and (retries % 3) == 0
|
|
211
|
-
sleep 5
|
|
212
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
|
213
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
|
214
|
-
sleep 10
|
|
215
|
-
retry
|
|
216
|
-
end while resp.state != "available"
|
|
217
|
-
end
|
|
218
|
-
rescue NoMethodError => e
|
|
219
|
-
if retries <= 3
|
|
220
|
-
MU.log "Got bogus Aws::EmptyResponse error on #{subnet_id} (retries used: #{retries}/3)", MU::WARN
|
|
221
|
-
retries = retries + 1
|
|
222
|
-
sleep 5
|
|
223
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
|
224
|
-
retry
|
|
225
|
-
else
|
|
226
|
-
raise e
|
|
227
|
-
end
|
|
228
|
-
end
|
|
229
|
-
|
|
230
|
-
if !subnet['route_table'].nil?
|
|
231
|
-
routes = {}
|
|
232
|
-
@config['route_tables'].each { |tbl|
|
|
233
|
-
routes[tbl['name']] = tbl
|
|
234
|
-
}
|
|
235
|
-
if routes.nil? or routes[subnet['route_table']].nil?
|
|
236
|
-
MU.log "Subnet #{subnet_name} references non-existent route #{subnet['route_table']}", MU::ERR, details: @deploy.deployment['vpcs']
|
|
237
|
-
raise MuError, "deploy failure"
|
|
238
|
-
end
|
|
239
|
-
MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
|
|
240
|
-
retries = 0
|
|
241
|
-
begin
|
|
242
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
|
|
243
|
-
route_table_id: routes[subnet['route_table']]['route_table_id'],
|
|
244
|
-
subnet_id: subnet_id
|
|
245
|
-
)
|
|
246
|
-
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound => e
|
|
247
|
-
retries = retries + 1
|
|
248
|
-
if retries < 10
|
|
249
|
-
sleep 10
|
|
250
|
-
retry
|
|
251
|
-
else
|
|
252
|
-
raise MuError, e.inspect
|
|
253
|
-
end
|
|
254
|
-
end
|
|
255
|
-
end
|
|
256
|
-
retries = 0
|
|
257
|
-
begin
|
|
258
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
|
259
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
|
260
|
-
if retries < 10
|
|
261
|
-
MU.log "Got #{e.inspect}, waiting and retrying", MU::WARN
|
|
262
|
-
sleep 10
|
|
263
|
-
retries = retries + 1
|
|
264
|
-
retry
|
|
265
|
-
end
|
|
266
|
-
raise MuError, e.inspect, e.backtrace
|
|
267
|
-
end
|
|
268
|
-
|
|
269
|
-
if subnet['is_public'] && subnet['create_nat_gateway']
|
|
270
|
-
MU::MommaCat.lock("nat-gateway-eipalloc")
|
|
271
|
-
filters = [{name: "domain", values: ["vpc"]}]
|
|
272
|
-
eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(filters: filters).addresses
|
|
273
|
-
allocation_id = nil
|
|
274
|
-
eips.each { |eip|
|
|
275
|
-
next if !eip.association_id.nil? and !eip.association_id.empty?
|
|
276
|
-
if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
|
|
277
|
-
if !allocation_ids.include?(eip.allocation_id)
|
|
278
|
-
allocation_id = eip.allocation_id
|
|
279
|
-
break
|
|
280
|
-
end
|
|
281
|
-
end
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
if allocation_id.nil?
|
|
285
|
-
allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
|
|
286
|
-
MU::MommaCat.lock(allocation_id, false, true)
|
|
287
|
-
end
|
|
131
|
+
nat_gateways = create_subnets
|
|
288
132
|
|
|
289
|
-
|
|
290
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
|
|
291
|
-
subnet_id: subnet['subnet_id'],
|
|
292
|
-
allocation_id: allocation_id,
|
|
293
|
-
).nat_gateway
|
|
294
|
-
|
|
295
|
-
nat_gateway_id = resp.nat_gateway_id
|
|
296
|
-
attempts = 0
|
|
297
|
-
MU::MommaCat.unlock("nat-gateway-eipalloc")
|
|
298
|
-
while resp.class.name != "Aws::EC2::Types::NatGateway" or resp.state == "pending"
|
|
299
|
-
MU.log "Waiting for nat gateway #{nat_gateway_id} () to become available (EIP allocation: #{allocation_id})" if attempts % 5 == 0
|
|
300
|
-
sleep 30
|
|
301
|
-
begin
|
|
302
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
|
|
303
|
-
rescue Aws::EmptyStructure, NoMethodError
|
|
304
|
-
sleep 5
|
|
305
|
-
retry
|
|
306
|
-
end
|
|
307
|
-
if attempts > 30
|
|
308
|
-
MU::MommaCat.unlock(allocation_id, true)
|
|
309
|
-
raise MuError, "Timed out while waiting for NAT Gateway #{nat_gateway_id}: #{resp}"
|
|
310
|
-
end
|
|
311
|
-
attempts += 1
|
|
312
|
-
end
|
|
313
|
-
MU::MommaCat.unlock(allocation_id, true)
|
|
314
|
-
|
|
315
|
-
raise MuError, "NAT Gateway failed #{nat_gateway_id}: #{resp}" if resp.state == "failed"
|
|
316
|
-
nat_gateways << {'id' => nat_gateway_id, 'availability_zone' => subnet['availability_zone']}
|
|
317
|
-
@tags.each_pair { |k, v|
|
|
318
|
-
MU::Cloud::AWS.createTag(nat_gateway_id, k, v, region: @config['region'], credentials: @config['credentials'])
|
|
319
|
-
}
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
if subnet.has_key?("map_public_ips")
|
|
323
|
-
retries = 0
|
|
324
|
-
begin
|
|
325
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
|
|
326
|
-
subnet_id: subnet_id,
|
|
327
|
-
map_public_ip_on_launch: {
|
|
328
|
-
value: subnet['map_public_ips'],
|
|
329
|
-
}
|
|
330
|
-
)
|
|
331
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
|
332
|
-
if retries < 10
|
|
333
|
-
MU.log "Got #{e.inspect} while trying to enable map_public_ips on subnet, waiting and retrying", MU::WARN
|
|
334
|
-
sleep 10
|
|
335
|
-
retries += 1
|
|
336
|
-
retry
|
|
337
|
-
end
|
|
338
|
-
raise MuError, "Got #{e.inspect}, #{e.backtrace} while trying to enable map_public_ips on subnet"
|
|
339
|
-
end
|
|
340
|
-
end
|
|
341
|
-
|
|
342
|
-
if subnet["enable_traffic_logging"]
|
|
343
|
-
loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
|
|
344
|
-
logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
|
|
345
|
-
MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
|
|
346
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
|
|
347
|
-
resource_ids: [subnet_id],
|
|
348
|
-
resource_type: "Subnet",
|
|
349
|
-
traffic_type: subnet["traffic_type_to_log"],
|
|
350
|
-
log_group_name: loggroup.mu_name,
|
|
351
|
-
deliver_logs_permission_arn: logrole.cloudobj.arn
|
|
352
|
-
)
|
|
353
|
-
end
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
subnetthreads.each { |t|
|
|
358
|
-
t.join
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
notify
|
|
362
|
-
end
|
|
133
|
+
notify
|
|
363
134
|
|
|
364
135
|
if !nat_gateways.empty?
|
|
365
136
|
nat_gateways.each { |gateway|
|
|
366
137
|
@config['subnets'].each { |subnet|
|
|
367
|
-
if subnet['is_public']
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
else
|
|
388
|
-
raise e
|
|
389
|
-
end
|
|
390
|
-
rescue Aws::EC2::Errors::RouteAlreadyExists => e
|
|
391
|
-
MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
|
|
392
|
-
end
|
|
393
|
-
end
|
|
394
|
-
}
|
|
395
|
-
end
|
|
138
|
+
next if subnet['is_public'] != false or subnet['availability_zone'] != gateway['availability_zone']
|
|
139
|
+
|
|
140
|
+
@config['route_tables'].each { |rtb|
|
|
141
|
+
next if rtb['name'] != subnet['route_table']
|
|
142
|
+
rtb['routes'].each { |route|
|
|
143
|
+
next if route['gateway'] != '#NAT'
|
|
144
|
+
route_config = {
|
|
145
|
+
:route_table_id => rtb['route_table_id'],
|
|
146
|
+
:destination_cidr_block => route['destination_network'],
|
|
147
|
+
:nat_gateway_id => gateway['id']
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
|
|
151
|
+
MU.retrier([Aws::EC2::Errors::InvalidNatGatewayIDNotFound], wait: 10, max: 5) {
|
|
152
|
+
begin
|
|
153
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
|
|
154
|
+
rescue Aws::EC2::Errors::RouteAlreadyExists
|
|
155
|
+
MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
|
|
156
|
+
end
|
|
157
|
+
}
|
|
396
158
|
}
|
|
397
|
-
|
|
159
|
+
}
|
|
398
160
|
}
|
|
399
161
|
}
|
|
400
162
|
end
|
|
@@ -402,14 +164,14 @@ module MU
|
|
|
402
164
|
if @config['enable_dns_support']
|
|
403
165
|
MU.log "Enabling DNS support in #{@mu_name}"
|
|
404
166
|
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
|
|
405
|
-
vpc_id:
|
|
167
|
+
vpc_id: @cloud_id,
|
|
406
168
|
enable_dns_support: {value: @config['enable_dns_support']}
|
|
407
169
|
)
|
|
408
170
|
end
|
|
409
171
|
if @config['enable_dns_hostnames']
|
|
410
172
|
MU.log "Enabling DNS hostnames in #{@mu_name}"
|
|
411
173
|
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
|
|
412
|
-
vpc_id:
|
|
174
|
+
vpc_id: @cloud_id,
|
|
413
175
|
enable_dns_hostnames: {value: @config['enable_dns_hostnames']}
|
|
414
176
|
)
|
|
415
177
|
end
|
|
@@ -438,29 +200,16 @@ module MU
|
|
|
438
200
|
dhcp_configurations: dhcpopts
|
|
439
201
|
)
|
|
440
202
|
dhcpopt_id = resp.dhcp_options.dhcp_options_id
|
|
441
|
-
|
|
442
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
|
443
|
-
|
|
444
|
-
if @config['tags']
|
|
445
|
-
@config['tags'].each { |tag|
|
|
446
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
447
|
-
}
|
|
448
|
-
end
|
|
449
|
-
|
|
450
|
-
if @config['optional_tags']
|
|
451
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
452
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
453
|
-
}
|
|
454
|
-
end
|
|
203
|
+
tag_me(dhcpopt_id)
|
|
455
204
|
|
|
456
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id:
|
|
205
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: @cloud_id)
|
|
457
206
|
end
|
|
458
207
|
notify
|
|
459
208
|
|
|
460
209
|
if !MU::Cloud::AWS.isGovCloud?(@config['region'])
|
|
461
210
|
mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
|
|
462
211
|
if !mu_zone.nil?
|
|
463
|
-
MU::Cloud
|
|
212
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
|
|
464
213
|
end
|
|
465
214
|
end
|
|
466
215
|
loadSubnets
|
|
@@ -471,9 +220,6 @@ module MU
|
|
|
471
220
|
# Canonical Amazon Resource Number for this resource
|
|
472
221
|
# @return [String]
|
|
473
222
|
def arn
|
|
474
|
-
puts @config['region']
|
|
475
|
-
puts MU::Cloud::AWS.credToAcct(@config['credentials'])
|
|
476
|
-
puts @cloud_id
|
|
477
223
|
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":vpc/"+@cloud_id
|
|
478
224
|
end
|
|
479
225
|
|
|
@@ -490,202 +236,7 @@ module MU
|
|
|
490
236
|
# Generate peering connections
|
|
491
237
|
if !@config['peers'].nil? and @config['peers'].size > 0
|
|
492
238
|
@config['peers'].each { |peer|
|
|
493
|
-
|
|
494
|
-
peer_id = nil
|
|
495
|
-
peer['name'] ||= peer['vpc_name']
|
|
496
|
-
peer['id'] ||= peer['vpc_id']
|
|
497
|
-
|
|
498
|
-
# If we know this to be a sibling VPC elsewhere in our stack,
|
|
499
|
-
# go fetch it, and fix it if we've been misconfigured with a
|
|
500
|
-
# duplicate peering connection
|
|
501
|
-
if peer['vpc']['name'] and !peer['account']
|
|
502
|
-
peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs")
|
|
503
|
-
if peer_obj
|
|
504
|
-
if peer_obj.config['peers']
|
|
505
|
-
skipme = false
|
|
506
|
-
peer_obj.config['peers'].each { |peerpeer|
|
|
507
|
-
if peerpeer['vpc']['name'] == @config['name'] and
|
|
508
|
-
(peer['vpc']['name'] <=> @config['name']) == -1
|
|
509
|
-
skipme = true
|
|
510
|
-
MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
|
|
511
|
-
# XXX and if deploy_id matches or is unset
|
|
512
|
-
end
|
|
513
|
-
}
|
|
514
|
-
end
|
|
515
|
-
next if skipme
|
|
516
|
-
peer['account'] = MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
|
517
|
-
peer['vpc']['id'] = peer_obj.cloud_id
|
|
518
|
-
peer['vpc']['region'] ||= peer_obj.config['region']
|
|
519
|
-
end
|
|
520
|
-
end
|
|
521
|
-
|
|
522
|
-
# If we still don't know our peer's vpc identifier, go fishing
|
|
523
|
-
if !peer_obj
|
|
524
|
-
tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
|
|
525
|
-
if peer['vpc']['deploy_id'].nil? and peer['vpc']['id'].nil? and tag_key.nil?
|
|
526
|
-
peer['vpc']['deploy_id'] = @deploy.deploy_id
|
|
527
|
-
end
|
|
528
|
-
peer_obj = MU::MommaCat.findStray(
|
|
529
|
-
"AWS",
|
|
530
|
-
"vpcs",
|
|
531
|
-
deploy_id: peer['vpc']['deploy_id'],
|
|
532
|
-
cloud_id: peer['vpc']['id'],
|
|
533
|
-
# XXX we need a credentials argument here... maybe
|
|
534
|
-
name: peer['vpc']['name'],
|
|
535
|
-
tag_key: tag_key,
|
|
536
|
-
tag_value: tag_value,
|
|
537
|
-
dummy_ok: true,
|
|
538
|
-
region: peer['vpc']['region']
|
|
539
|
-
)
|
|
540
|
-
MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
|
|
541
|
-
raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
|
|
542
|
-
peer_obj = peer_obj.first
|
|
543
|
-
peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
|
544
|
-
peer['vpc']['id'] ||= peer_obj.cloud_id
|
|
545
|
-
peer['vpc']['region'] ||= peer_obj.config['region']
|
|
546
|
-
end
|
|
547
|
-
|
|
548
|
-
peer_id = peer['vpc']['id']
|
|
549
|
-
peer['account'] ||= MU::Cloud::AWS.account_number
|
|
550
|
-
|
|
551
|
-
# See if the peering connection exists before we bother
|
|
552
|
-
# creating it.
|
|
553
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
554
|
-
filters: [
|
|
555
|
-
{
|
|
556
|
-
name: "requester-vpc-info.vpc-id",
|
|
557
|
-
values: [@cloud_id]
|
|
558
|
-
},
|
|
559
|
-
{
|
|
560
|
-
name: "accepter-vpc-info.vpc-id",
|
|
561
|
-
values: [peer_id.to_s]
|
|
562
|
-
}
|
|
563
|
-
]
|
|
564
|
-
)
|
|
565
|
-
|
|
566
|
-
peering_id = if !resp or !resp.vpc_peering_connections or
|
|
567
|
-
resp.vpc_peering_connections.empty?
|
|
568
|
-
|
|
569
|
-
MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
|
|
570
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
|
|
571
|
-
vpc_id: @cloud_id,
|
|
572
|
-
peer_vpc_id: peer_id,
|
|
573
|
-
peer_owner_id: peer['account'],
|
|
574
|
-
peer_region: peer['vpc']['region']
|
|
575
|
-
)
|
|
576
|
-
resp.vpc_peering_connection.vpc_peering_connection_id
|
|
577
|
-
else
|
|
578
|
-
resp.vpc_peering_connections.first.vpc_peering_connection_id
|
|
579
|
-
end
|
|
580
|
-
|
|
581
|
-
peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer['vpc']['id'])
|
|
582
|
-
|
|
583
|
-
MU::Cloud::AWS.createStandardTags(peering_id, region: @config['region'], credentials: @config['credentials'])
|
|
584
|
-
MU::Cloud::AWS.createTag(peering_id, "Name", peering_name, region: @config['region'], credentials: @config['credentials'])
|
|
585
|
-
|
|
586
|
-
if @config['optional_tags']
|
|
587
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
588
|
-
MU::Cloud::AWS.createTag(peering_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
589
|
-
}
|
|
590
|
-
end
|
|
591
|
-
|
|
592
|
-
if @config['tags']
|
|
593
|
-
@config['tags'].each { |tag|
|
|
594
|
-
MU::Cloud::AWS.createTag(peering_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
595
|
-
}
|
|
596
|
-
end
|
|
597
|
-
|
|
598
|
-
# Create routes to our new friend.
|
|
599
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
|
|
600
|
-
my_route_config = {
|
|
601
|
-
:route_table_id => rtb_id,
|
|
602
|
-
:destination_cidr_block => peer_obj.cloud_desc.cidr_block,
|
|
603
|
-
:vpc_peering_connection_id => peering_id
|
|
604
|
-
}
|
|
605
|
-
rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
606
|
-
route_table_ids: [rtb_id]
|
|
607
|
-
).route_tables.first
|
|
608
|
-
already_exists = false
|
|
609
|
-
rtbdesc.routes.each { |r|
|
|
610
|
-
if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
|
|
611
|
-
if r.vpc_peering_connection_id != peering_id
|
|
612
|
-
MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
|
|
613
|
-
raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
|
|
614
|
-
else
|
|
615
|
-
already_exists = true
|
|
616
|
-
end
|
|
617
|
-
end
|
|
618
|
-
}
|
|
619
|
-
next if already_exists
|
|
620
|
-
|
|
621
|
-
MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
|
|
622
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
|
|
623
|
-
} # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
|
|
624
|
-
|
|
625
|
-
begin
|
|
626
|
-
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
627
|
-
vpc_peering_connection_ids: [peering_id]
|
|
628
|
-
).vpc_peering_connections.first
|
|
629
|
-
|
|
630
|
-
if cnxn.status.code == "pending-acceptance"
|
|
631
|
-
if ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
|
|
632
|
-
MU.log "Auto-accepting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
|
|
633
|
-
begin
|
|
634
|
-
MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).accept_vpc_peering_connection(
|
|
635
|
-
vpc_peering_connection_id: peering_id,
|
|
636
|
-
)
|
|
637
|
-
if peer['account'] != MU::Cloud::AWS.credToAcct(@config['credentials'])
|
|
638
|
-
# this seems to take a while across accounts
|
|
639
|
-
sleep 5
|
|
640
|
-
end
|
|
641
|
-
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
642
|
-
vpc_peering_connection_ids: [peering_id]
|
|
643
|
-
).vpc_peering_connections.first
|
|
644
|
-
rescue Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists => e
|
|
645
|
-
MU.log "Attempt to create duplicate peering connection to #{peer_id} from VPC #{@config['name']}", MU::WARN
|
|
646
|
-
end
|
|
647
|
-
|
|
648
|
-
# Create routes back from our new friend to us.
|
|
649
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer['vpc']['region'], credentials: peer['account']).uniq.each { |rtb_id|
|
|
650
|
-
peer_route_config = {
|
|
651
|
-
:route_table_id => rtb_id,
|
|
652
|
-
:destination_cidr_block => @config['ip_block'],
|
|
653
|
-
:vpc_peering_connection_id => peering_id
|
|
654
|
-
}
|
|
655
|
-
begin
|
|
656
|
-
resp = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).create_route(peer_route_config)
|
|
657
|
-
rescue Aws::EC2::Errors::RouteAlreadyExists => e
|
|
658
|
-
rtbdesc = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).describe_route_tables(
|
|
659
|
-
route_table_ids: [rtb_id]
|
|
660
|
-
).route_tables.first
|
|
661
|
-
rtbdesc.routes.each { |r|
|
|
662
|
-
if r.destination_cidr_block == @config['ip_block']
|
|
663
|
-
if r.vpc_peering_connection_id != peering_id
|
|
664
|
-
MU.log "Attempt to create duplicate route to VPC #{@config['name']} (#{@config['ip_block']}) from peer VPC #{peer_id}'s route table #{rtb_id}", MU::ERR
|
|
665
|
-
end
|
|
666
|
-
end
|
|
667
|
-
}
|
|
668
|
-
end
|
|
669
|
-
}
|
|
670
|
-
else
|
|
671
|
-
MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
|
|
672
|
-
end
|
|
673
|
-
end
|
|
674
|
-
|
|
675
|
-
if cnxn.status.code == "failed" or cnxn.status.code == "rejected" or cnxn.status.code == "expired" or cnxn.status.code == "deleted"
|
|
676
|
-
MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer['vpc']['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
|
|
677
|
-
begin
|
|
678
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
|
|
679
|
-
vpc_peering_connection_id: peering_id
|
|
680
|
-
)
|
|
681
|
-
rescue Aws::EC2::Errors::InvalidStateTransition => e
|
|
682
|
-
# XXX apparently this is normal?
|
|
683
|
-
end
|
|
684
|
-
raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
|
|
685
|
-
end
|
|
686
|
-
|
|
687
|
-
end while cnxn.status.code != "active" and !((peer_obj.nil? or peer_obj.deploydata.nil? or !peer_obj.deploydata['auto_accept_peers']) and cnxn.status.code == "pending-acceptance")
|
|
688
|
-
|
|
239
|
+
peerWith(peer)
|
|
689
240
|
}
|
|
690
241
|
end
|
|
691
242
|
|
|
@@ -837,12 +388,9 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
|
|
|
837
388
|
if rtb_desc.associations
|
|
838
389
|
rtb_desc.associations.each { |assoc|
|
|
839
390
|
if assoc.subnet_id
|
|
840
|
-
if associations[assoc.subnet_id] and associations[assoc.subnet_id] != rtb['name']
|
|
841
|
-
MU.log "wait more than one route table association for #{assoc.subnet_id} what", MU::WARN, details: associations[assoc.subnet_id]+" => "+rtb['name']
|
|
842
|
-
end
|
|
843
391
|
associations[assoc.subnet_id] = rtb['name']
|
|
844
392
|
elsif assoc.gateway_id
|
|
845
|
-
MU.log "association I don't
|
|
393
|
+
MU.log " Saw a route table association I don't know how to adopt in #{@cloud_id}", MU::WARN, details: rtb_desc
|
|
846
394
|
end
|
|
847
395
|
}
|
|
848
396
|
end
|
|
@@ -902,20 +450,18 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
902
450
|
# Describe subnets associated with this VPC. We'll compose identifying
|
|
903
451
|
# information similar to what MU::Cloud.describe builds for first-class
|
|
904
452
|
# resources.
|
|
905
|
-
#
|
|
906
|
-
# that work like first-class objects. How would we enforce that?
|
|
907
|
-
# @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
|
|
453
|
+
# @return [Array<MU::Cloud::AWS::VPC::Subnet>]
|
|
908
454
|
def loadSubnets
|
|
909
|
-
if
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
]
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
455
|
+
return [] if !@cloud_id
|
|
456
|
+
|
|
457
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
|
|
458
|
+
filters: [
|
|
459
|
+
{ name: "vpc-id", values: [@cloud_id] }
|
|
460
|
+
]
|
|
461
|
+
)
|
|
462
|
+
if resp.nil? or resp.subnets.nil? or resp.subnets.empty?
|
|
463
|
+
MU.log "Got empty results when trying to list subnets in #{@cloud_id}", MU::WARN
|
|
464
|
+
return []
|
|
919
465
|
end
|
|
920
466
|
|
|
921
467
|
@subnetcachesemaphore.synchronize {
|
|
@@ -926,23 +472,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
926
472
|
# metadata. Like ya do.
|
|
927
473
|
if !@config.nil? and @config.has_key?("subnets")
|
|
928
474
|
@config['subnets'].each { |subnet|
|
|
929
|
-
subnet['mu_name']
|
|
475
|
+
subnet['mu_name'] ||= @mu_name+"-"+subnet['name']
|
|
930
476
|
subnet['region'] = @config['region']
|
|
931
477
|
subnet['credentials'] = @config['credentials']
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
}
|
|
940
|
-
end
|
|
478
|
+
resp.subnets.each { |desc|
|
|
479
|
+
if desc.cidr_block == subnet["ip_block"]
|
|
480
|
+
subnet["tags"] = MU.structToHash(desc.tags)
|
|
481
|
+
subnet["cloud_id"] = desc.subnet_id
|
|
482
|
+
break
|
|
483
|
+
end
|
|
484
|
+
}
|
|
941
485
|
|
|
942
486
|
if subnet["cloud_id"] and !ext_ids.include?(subnet["cloud_id"])
|
|
943
487
|
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
|
944
488
|
elsif !subnet["cloud_id"]
|
|
945
|
-
resp.
|
|
489
|
+
resp.subnets.each { |desc|
|
|
946
490
|
if desc.cidr_block == subnet["ip_block"]
|
|
947
491
|
subnet['cloud_id'] = desc.subnet_id
|
|
948
492
|
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
|
@@ -955,21 +499,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
955
499
|
|
|
956
500
|
# Of course we might be loading up a dummy subnet object from a
|
|
957
501
|
# foreign or non-Mu-created VPC and subnet. So make something up.
|
|
958
|
-
if
|
|
959
|
-
resp.
|
|
960
|
-
subnet = {
|
|
961
|
-
|
|
962
|
-
|
|
502
|
+
if @subnets.empty?
|
|
503
|
+
resp.subnets.each { |desc|
|
|
504
|
+
subnet = {
|
|
505
|
+
"ip_block" => desc.cidr_block,
|
|
506
|
+
"tags" => MU.structToHash(desc.tags),
|
|
507
|
+
"cloud_id" => desc.subnet_id,
|
|
508
|
+
'region' => @config['region'],
|
|
509
|
+
'credentials' => @config['credentials'],
|
|
510
|
+
}
|
|
511
|
+
subnet['name'] = subnet["ip_block"].gsub(/[\.\/]/, "_")
|
|
963
512
|
subnet['mu_name'] = @mu_name+"-"+subnet['name']
|
|
964
|
-
|
|
965
|
-
subnet["cloud_id"] = desc.subnet_id
|
|
966
|
-
subnet['region'] = @config['region']
|
|
967
|
-
subnet['credentials'] = @config['credentials']
|
|
968
|
-
if !ext_ids.include?(desc.subnet_id)
|
|
969
|
-
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
|
970
|
-
end
|
|
513
|
+
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
|
971
514
|
}
|
|
972
515
|
end
|
|
516
|
+
|
|
973
517
|
return @subnets
|
|
974
518
|
}
|
|
975
519
|
end
|
|
@@ -1138,14 +682,14 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1138
682
|
MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, region: MU.myRegion)
|
|
1139
683
|
MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, region: region, credentials: credentials)
|
|
1140
684
|
|
|
1141
|
-
if MU::Cloud::AWS::VPC.
|
|
685
|
+
if MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
|
|
1142
686
|
return true
|
|
1143
687
|
else
|
|
1144
688
|
# The cache can be out of date at times, check again without it
|
|
1145
689
|
MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, use_cache: false, region: MU.myRegion)
|
|
1146
690
|
MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, use_cache: false, region: region, credentials: credentials)
|
|
1147
691
|
|
|
1148
|
-
return MU::Cloud::AWS::VPC.
|
|
692
|
+
return MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
|
|
1149
693
|
end
|
|
1150
694
|
|
|
1151
695
|
end
|
|
@@ -1184,7 +728,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1184
728
|
# @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
|
|
1185
729
|
# @param instance_id [String]: The instance ID in the target subnet/subnets.
|
|
1186
730
|
# @return [Boolean]
|
|
1187
|
-
def self.
|
|
731
|
+
def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
|
|
1188
732
|
my_routes = []
|
|
1189
733
|
vpc_peer_mapping = {}
|
|
1190
734
|
|
|
@@ -1278,32 +822,39 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1278
822
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
1279
823
|
# @param region [String]: The cloud provider region
|
|
1280
824
|
# @return [void]
|
|
1281
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
825
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
1282
826
|
MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
1283
827
|
|
|
1284
828
|
tagfilters = [
|
|
1285
|
-
{name: "tag:MU-ID", values: [
|
|
829
|
+
{name: "tag:MU-ID", values: [deploy_id]}
|
|
1286
830
|
]
|
|
1287
831
|
if !ignoremaster
|
|
1288
832
|
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
|
1289
833
|
end
|
|
1290
834
|
|
|
1291
835
|
vpcs = []
|
|
1292
|
-
|
|
1293
|
-
begin
|
|
836
|
+
MU.retrier([Aws::EC2::Errors::InvalidVpcIDNotFound], wait: 5) {
|
|
1294
837
|
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters, max_results: 1000).vpcs
|
|
1295
838
|
vpcs = resp if !resp.empty?
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
# resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
842
|
+
# filters: [
|
|
843
|
+
# {
|
|
844
|
+
# name: "requester-vpc-info.vpc-id",
|
|
845
|
+
# values: [@cloud_id]
|
|
846
|
+
# },
|
|
847
|
+
# {
|
|
848
|
+
# name: "accepter-vpc-info.vpc-id",
|
|
849
|
+
# values: [peer_id.to_s]
|
|
850
|
+
# }
|
|
851
|
+
# ]
|
|
852
|
+
# )
|
|
1303
853
|
|
|
1304
854
|
if !vpcs.empty?
|
|
1305
855
|
gwthreads = []
|
|
1306
856
|
vpcs.each { |vpc|
|
|
857
|
+
purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
|
|
1307
858
|
# NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
|
|
1308
859
|
gwthreads << Thread.new {
|
|
1309
860
|
purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
|
|
@@ -1325,7 +876,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1325
876
|
|
|
1326
877
|
# unless noop
|
|
1327
878
|
# MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
|
|
1328
|
-
# match_string = "#{
|
|
879
|
+
# match_string = "#{deploy_id}.*TRAFFIC-LOG"
|
|
1329
880
|
# }
|
|
1330
881
|
# end
|
|
1331
882
|
end
|
|
@@ -1379,11 +930,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1379
930
|
logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
|
|
1380
931
|
# logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
|
|
1381
932
|
configurator.insertKitten(logdesc, "logs")
|
|
1382
|
-
vpc['
|
|
1383
|
-
vpc['dependencies'] << {
|
|
1384
|
-
"type" => "log",
|
|
1385
|
-
"name" => vpc['name']+"loggroup"
|
|
1386
|
-
}
|
|
933
|
+
MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
|
|
1387
934
|
|
|
1388
935
|
roledesc = {
|
|
1389
936
|
"name" => vpc['name']+"logrole",
|
|
@@ -1421,11 +968,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1421
968
|
roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
|
|
1422
969
|
roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
|
|
1423
970
|
configurator.insertKitten(roledesc, "roles")
|
|
1424
|
-
vpc['
|
|
1425
|
-
vpc['dependencies'] << {
|
|
1426
|
-
"type" => "role",
|
|
1427
|
-
"name" => vpc['name']+"logrole"
|
|
1428
|
-
}
|
|
971
|
+
MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
|
|
1429
972
|
end
|
|
1430
973
|
|
|
1431
974
|
subnet_routes = Hash.new
|
|
@@ -1476,10 +1019,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1476
1019
|
subnet_routes[table['name']].each { |subnet|
|
|
1477
1020
|
nat_routes[subnet] = route['nat_host_name']
|
|
1478
1021
|
}
|
|
1479
|
-
vpc['
|
|
1480
|
-
"type" => "server",
|
|
1481
|
-
"name" => route['nat_host_name']
|
|
1482
|
-
}
|
|
1022
|
+
MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
|
|
1483
1023
|
elsif route['gateway'] == '#NAT'
|
|
1484
1024
|
vpc['create_nat_gateway'] = true
|
|
1485
1025
|
private_rtbs << table['name']
|
|
@@ -1492,17 +1032,11 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1492
1032
|
if route['gateway'] == '#INTERNET'
|
|
1493
1033
|
if table['name'] == subnet['route_table']
|
|
1494
1034
|
subnet['is_public'] = true
|
|
1495
|
-
if vpc['create_nat_gateway']
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
subnet['create_nat_gateway'] = false
|
|
1501
|
-
else
|
|
1502
|
-
subnet['create_nat_gateway'] = true
|
|
1503
|
-
nat_gateway_added = true
|
|
1504
|
-
end
|
|
1505
|
-
end
|
|
1035
|
+
if vpc['create_nat_gateway'] and (vpc['nat_gateway_multi_az'] or !nat_gateway_added)
|
|
1036
|
+
subnet['create_nat_gateway'] = true
|
|
1037
|
+
nat_gateway_added = true
|
|
1038
|
+
else
|
|
1039
|
+
subnet['create_nat_gateway'] = false
|
|
1506
1040
|
end
|
|
1507
1041
|
else
|
|
1508
1042
|
subnet['is_public'] = false
|
|
@@ -1662,17 +1196,10 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1662
1196
|
|
|
1663
1197
|
ifaces.each { |iface|
|
|
1664
1198
|
if iface.vpc_id
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
]
|
|
1670
|
-
).security_groups
|
|
1671
|
-
if default_sg_resp and default_sg_resp.size == 1
|
|
1672
|
-
default_sg = default_sg_resp.first.group_id
|
|
1673
|
-
if iface.groups.size > 1 or
|
|
1674
|
-
iface.groups.first.group_id != default_sg
|
|
1675
|
-
MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
|
|
1199
|
+
default_sg = MU::Cloud::AWS::VPC.getDefaultSg(iface.vpc_id, region: region, credentials: credentials)
|
|
1200
|
+
if default_sg and (iface.groups.size > 1 or (iface.groups.size == 1 and iface.groups.first.group_id != default_sg))
|
|
1201
|
+
MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
|
|
1202
|
+
if !noop
|
|
1676
1203
|
begin
|
|
1677
1204
|
MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
|
|
1678
1205
|
network_interface_id: iface.network_interface_id,
|
|
@@ -1693,11 +1220,15 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1693
1220
|
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
|
1694
1221
|
MU.log "Can't detach #{iface.network_interface_id}: #{e.message}", MU::WARN, details: iface.attachment
|
|
1695
1222
|
next
|
|
1223
|
+
rescue Aws::EC2::Errors::IncorrectState => e
|
|
1224
|
+
MU.log e.message, MU::WARN
|
|
1225
|
+
sleep 5
|
|
1226
|
+
retry
|
|
1696
1227
|
rescue Aws::EC2::Errors::InvalidAttachmentIDNotFound => e
|
|
1697
1228
|
# suits me just fine
|
|
1698
1229
|
rescue Aws::EC2::Errors::AuthFailure => e
|
|
1699
1230
|
if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
|
|
1700
|
-
MU::Cloud
|
|
1231
|
+
MU::Cloud.resourceClass("AWS", "LoadBalancer").cleanup(
|
|
1701
1232
|
noop: noop,
|
|
1702
1233
|
region: region,
|
|
1703
1234
|
credentials: credentials,
|
|
@@ -1719,9 +1250,230 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1719
1250
|
}
|
|
1720
1251
|
end
|
|
1721
1252
|
|
|
1253
|
+
# Fetch the group id of the +default+ security group for the given VPC
|
|
1254
|
+
# @param vpc_id [String]
|
|
1255
|
+
# @param region [String]
|
|
1256
|
+
# @param credentials [String]
|
|
1257
|
+
# @return [String]
|
|
1258
|
+
def self.getDefaultSg(vpc_id, region: MU.curRegion, credentials: nil)
|
|
1259
|
+
default_sg_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
|
|
1260
|
+
filters: [
|
|
1261
|
+
{ name: "group-name", values: ["default"] },
|
|
1262
|
+
{ name: "vpc-id", values: [vpc_id] }
|
|
1263
|
+
]
|
|
1264
|
+
).security_groups
|
|
1265
|
+
if default_sg_resp and default_sg_resp.size == 1
|
|
1266
|
+
return default_sg_resp.first.group_id
|
|
1267
|
+
end
|
|
1268
|
+
nil
|
|
1269
|
+
end
|
|
1270
|
+
|
|
1271
|
+
# Try to locate the default VPC for a region, and return a BoK-style
|
|
1272
|
+
# config fragment for something that might want to live in it.
|
|
1273
|
+
def self.defaultVpc(region, credentials)
|
|
1274
|
+
cfg_fragment = nil
|
|
1275
|
+
MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs.vpcs.each { |vpc|
|
|
1276
|
+
if vpc.is_default
|
|
1277
|
+
cfg_fragment = {
|
|
1278
|
+
"id" => vpc.vpc_id,
|
|
1279
|
+
"cloud" => "AWS",
|
|
1280
|
+
"region" => region,
|
|
1281
|
+
"credentials" => credentials
|
|
1282
|
+
}
|
|
1283
|
+
cfg_fragment['subnets'] = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
|
|
1284
|
+
filters: [
|
|
1285
|
+
{
|
|
1286
|
+
name: "vpc-id",
|
|
1287
|
+
values: [vpc.vpc_id]
|
|
1288
|
+
}
|
|
1289
|
+
]
|
|
1290
|
+
).subnets.map { |s| { "subnet_id" => s.subnet_id } }
|
|
1291
|
+
break
|
|
1292
|
+
end
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
cfg_fragment
|
|
1296
|
+
end
|
|
1297
|
+
|
|
1298
|
+
# Return a {MU::Config::Ref} that indicates this VPC.
|
|
1299
|
+
# @param subnet_ids [Array<String>]: Optional list of subnet ids with which to infer a +subnet_pref+ parameter.
|
|
1300
|
+
# @return [MU::Config::Ref]
|
|
1301
|
+
def getReference(subnet_ids = [])
|
|
1302
|
+
have_private = have_public = false
|
|
1303
|
+
subnets.each { |s|
|
|
1304
|
+
next if subnet_ids and !subnet_ids.empty? and !subnet_ids.include?(s.cloud_id)
|
|
1305
|
+
if s.private?
|
|
1306
|
+
have_private = true
|
|
1307
|
+
else
|
|
1308
|
+
have_public = true
|
|
1309
|
+
end
|
|
1310
|
+
}
|
|
1311
|
+
subnet_pref = if have_private == have_public
|
|
1312
|
+
"any"
|
|
1313
|
+
elsif have_private
|
|
1314
|
+
"all_private"
|
|
1315
|
+
elsif have_public
|
|
1316
|
+
"all_public"
|
|
1317
|
+
end
|
|
1318
|
+
MU::Config::Ref.get(
|
|
1319
|
+
id: @cloud_id,
|
|
1320
|
+
cloud: "AWS",
|
|
1321
|
+
credentials: @credentials,
|
|
1322
|
+
region: @config['region'],
|
|
1323
|
+
type: "vpcs",
|
|
1324
|
+
subnet_pref: subnet_pref
|
|
1325
|
+
)
|
|
1326
|
+
end
|
|
1722
1327
|
|
|
1723
1328
|
private
|
|
1724
1329
|
|
|
1330
|
+
def peerWith(peer)
|
|
1331
|
+
peer_ref = MU::Config::Ref.get(peer['vpc'])
|
|
1332
|
+
peer_obj = peer_ref.kitten
|
|
1333
|
+
peer_id = peer_ref.kitten.cloud_id
|
|
1334
|
+
if peer_id == @cloud_id
|
|
1335
|
+
MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
|
|
1336
|
+
raise "#{@mu_name} attempted to peer with itself (#{@cloud_id})"
|
|
1337
|
+
end
|
|
1338
|
+
|
|
1339
|
+
if peer_obj and peer_obj.config['peers']
|
|
1340
|
+
peer_obj.config['peers'].each { |peerpeer|
|
|
1341
|
+
if peerpeer['vpc']['name'] == @config['name'] and
|
|
1342
|
+
(peer['vpc']['name'] <=> @config['name']) == -1
|
|
1343
|
+
MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
|
|
1344
|
+
return
|
|
1345
|
+
# XXX and if deploy_id matches or is unset
|
|
1346
|
+
end
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
|
1350
|
+
end
|
|
1351
|
+
|
|
1352
|
+
peer['account'] ||= MU::Cloud::AWS.account_number
|
|
1353
|
+
|
|
1354
|
+
# See if the peering connection exists before we bother
|
|
1355
|
+
# creating it.
|
|
1356
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
1357
|
+
filters: [
|
|
1358
|
+
{
|
|
1359
|
+
name: "requester-vpc-info.vpc-id",
|
|
1360
|
+
values: [@cloud_id]
|
|
1361
|
+
},
|
|
1362
|
+
{
|
|
1363
|
+
name: "accepter-vpc-info.vpc-id",
|
|
1364
|
+
values: [peer_id.to_s]
|
|
1365
|
+
}
|
|
1366
|
+
]
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
peering_id = if !resp or !resp.vpc_peering_connections or
|
|
1370
|
+
resp.vpc_peering_connections.empty?
|
|
1371
|
+
|
|
1372
|
+
MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
|
|
1373
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
|
|
1374
|
+
vpc_id: @cloud_id,
|
|
1375
|
+
peer_vpc_id: peer_id,
|
|
1376
|
+
peer_owner_id: peer['account'],
|
|
1377
|
+
peer_region: peer_obj.config['region']
|
|
1378
|
+
)
|
|
1379
|
+
resp.vpc_peering_connection.vpc_peering_connection_id
|
|
1380
|
+
else
|
|
1381
|
+
resp.vpc_peering_connections.first.vpc_peering_connection_id
|
|
1382
|
+
end
|
|
1383
|
+
|
|
1384
|
+
peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer_id)
|
|
1385
|
+
|
|
1386
|
+
tag_me(peering_id, peering_name)
|
|
1387
|
+
|
|
1388
|
+
# Create routes to our new friend.
|
|
1389
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
|
|
1390
|
+
my_route_config = {
|
|
1391
|
+
:route_table_id => rtb_id,
|
|
1392
|
+
:destination_cidr_block => peer_obj.cloud_desc.cidr_block,
|
|
1393
|
+
:vpc_peering_connection_id => peering_id
|
|
1394
|
+
}
|
|
1395
|
+
rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
1396
|
+
route_table_ids: [rtb_id]
|
|
1397
|
+
).route_tables.first
|
|
1398
|
+
already_exists = false
|
|
1399
|
+
rtbdesc.routes.each { |r|
|
|
1400
|
+
if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
|
|
1401
|
+
if r.vpc_peering_connection_id != peering_id
|
|
1402
|
+
MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
|
|
1403
|
+
raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
|
|
1404
|
+
else
|
|
1405
|
+
already_exists = true
|
|
1406
|
+
end
|
|
1407
|
+
end
|
|
1408
|
+
}
|
|
1409
|
+
next if already_exists
|
|
1410
|
+
|
|
1411
|
+
MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
|
|
1412
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
|
|
1413
|
+
} # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
|
|
1414
|
+
|
|
1415
|
+
can_auto_accept = ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
|
|
1416
|
+
|
|
1417
|
+
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
1418
|
+
vpc_peering_connection_ids: [peering_id]
|
|
1419
|
+
).vpc_peering_connections.first
|
|
1420
|
+
|
|
1421
|
+
loop_if = Proc.new {
|
|
1422
|
+
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
|
1423
|
+
vpc_peering_connection_ids: [peering_id]
|
|
1424
|
+
).vpc_peering_connections.first
|
|
1425
|
+
((can_auto_accept and cnxn.status.code == "pending-acceptance") or (cnxn.status.code != "active" and cnxn.status.code != "pending-acceptance"))
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
MU.retrier(wait: 5, loop_if: loop_if, ignoreme: [Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists, Aws::EC2::Errors::RouteAlreadyExists]) {
|
|
1429
|
+
if cnxn.status.code == "pending-acceptance"
|
|
1430
|
+
if can_auto_accept
|
|
1431
|
+
MU.log "Auto-accepting peering connection #{peering_id} from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
|
|
1432
|
+
MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).accept_vpc_peering_connection(
|
|
1433
|
+
vpc_peering_connection_id: peering_id,
|
|
1434
|
+
)
|
|
1435
|
+
|
|
1436
|
+
# Create routes back from our new friend to us.
|
|
1437
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer_obj.config['region'], credentials: peer['account']).uniq.each { |rtb_id|
|
|
1438
|
+
peer_route_config = {
|
|
1439
|
+
:route_table_id => rtb_id,
|
|
1440
|
+
:destination_cidr_block => @config['ip_block'],
|
|
1441
|
+
:vpc_peering_connection_id => peering_id
|
|
1442
|
+
}
|
|
1443
|
+
resp = MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).create_route(peer_route_config)
|
|
1444
|
+
}
|
|
1445
|
+
else
|
|
1446
|
+
MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
|
|
1447
|
+
end
|
|
1448
|
+
end
|
|
1449
|
+
|
|
1450
|
+
if ["failed", "rejected", "expired", "deleted"].include?(cnxn.status.code)
|
|
1451
|
+
MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer_obj.config['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
|
|
1452
|
+
begin
|
|
1453
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
|
|
1454
|
+
vpc_peering_connection_id: peering_id
|
|
1455
|
+
)
|
|
1456
|
+
rescue Aws::EC2::Errors::InvalidStateTransition
|
|
1457
|
+
# XXX apparently this is normal?
|
|
1458
|
+
end
|
|
1459
|
+
raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
|
|
1460
|
+
end
|
|
1461
|
+
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
def tag_me(resource_id = @cloud_id, name = @mu_name)
|
|
1467
|
+
MU::Cloud::AWS.createStandardTags(
|
|
1468
|
+
resource_id,
|
|
1469
|
+
region: @config['region'],
|
|
1470
|
+
credentials: @config['credentials'],
|
|
1471
|
+
optional: @config['optional_tags'],
|
|
1472
|
+
nametag: name,
|
|
1473
|
+
othertags: @config['tags']
|
|
1474
|
+
)
|
|
1475
|
+
end
|
|
1476
|
+
|
|
1725
1477
|
# Helper method for manufacturing route tables. Expect to be called from
|
|
1726
1478
|
# {MU::Cloud::AWS::VPC#create} or {MU::Cloud::AWS::VPC#groom}.
|
|
1727
1479
|
# @param rtb [Hash]: A route table description parsed through {MU::Config::BasketofKittens::vpcs::route_tables}.
|
|
@@ -1733,21 +1485,9 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1733
1485
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route_table(vpc_id: vpc_id).route_table
|
|
1734
1486
|
route_table_id = rtb['route_table_id'] = resp.route_table_id
|
|
1735
1487
|
sleep 5
|
|
1736
|
-
MU::Cloud::AWS.createTag(route_table_id, "Name", vpc_name+"-"+rtb['name'].upcase, credentials: @config['credentials'])
|
|
1737
|
-
|
|
1738
|
-
if @config['tags']
|
|
1739
|
-
@config['tags'].each { |tag|
|
|
1740
|
-
MU::Cloud::AWS.createTag(route_table_id, tag['key'], tag['value'], credentials: @config['credentials'])
|
|
1741
|
-
}
|
|
1742
|
-
end
|
|
1743
1488
|
|
|
1744
|
-
|
|
1745
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
1746
|
-
MU::Cloud::AWS.createTag(route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
1747
|
-
}
|
|
1748
|
-
end
|
|
1489
|
+
tag_me(route_table_id, vpc_name+"-"+rtb['name'].upcase)
|
|
1749
1490
|
|
|
1750
|
-
MU::Cloud::AWS.createStandardTags(route_table_id, credentials: @config['credentials'])
|
|
1751
1491
|
rtb['routes'].each { |route|
|
|
1752
1492
|
if route['nat_host_id'].nil? and route['nat_host_name'].nil?
|
|
1753
1493
|
route_config = {
|
|
@@ -1840,36 +1580,25 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1840
1580
|
).nat_gateways
|
|
1841
1581
|
|
|
1842
1582
|
threads = []
|
|
1843
|
-
|
|
1583
|
+
|
|
1844
1584
|
if !gateways.empty?
|
|
1845
1585
|
gateways.each { |gateway|
|
|
1586
|
+
next if noop
|
|
1587
|
+
MU.log "Deleting NAT Gateway #{gateway.nat_gateway_id}"
|
|
1846
1588
|
threads << Thread.new {
|
|
1847
|
-
MU.
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1589
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
|
|
1590
|
+
|
|
1591
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
|
1592
|
+
|
|
1593
|
+
loop_if = Proc.new {
|
|
1594
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
|
1595
|
+
(resp.state != "deleted" and resp.state != "failed")
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::NatGatewayMalformed, Aws::EC2::Errors::NatGatewayNotFound], max: 50, loop_if: loop_if) { |retries, _wait|
|
|
1599
|
+
MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if retries % 3 == 0
|
|
1600
|
+
}
|
|
1853
1601
|
|
|
1854
|
-
attempts = 0
|
|
1855
|
-
while resp.state != "deleted" and resp.state != "failed"
|
|
1856
|
-
MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if attempts % 2 == 0
|
|
1857
|
-
sleep 30
|
|
1858
|
-
begin
|
|
1859
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
|
1860
|
-
rescue Aws::EmptyStructure, NoMethodError
|
|
1861
|
-
sleep 5
|
|
1862
|
-
retry
|
|
1863
|
-
rescue Aws::EC2::Errors::NatGatewayNotFound
|
|
1864
|
-
MU.log "NAT gateway #{gateway.nat_gateway_id} already deleted", MU::NOTICE
|
|
1865
|
-
end
|
|
1866
|
-
MU.log "Timed out while waiting for NAT Gateway to delete #{gateway.nat_gateway_id}: #{resp}", MU::WARN if attempts > 50
|
|
1867
|
-
attempts += 1
|
|
1868
|
-
end
|
|
1869
|
-
rescue Aws::EC2::Errors::NatGatewayMalformed
|
|
1870
|
-
MU.log "NAT Gateway #{gateway.nat_gateway_id} was already deleted", MU::NOTICE
|
|
1871
|
-
end
|
|
1872
|
-
end
|
|
1873
1602
|
}
|
|
1874
1603
|
}
|
|
1875
1604
|
end
|
|
@@ -1898,38 +1627,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
1898
1627
|
).vpc_endpoints
|
|
1899
1628
|
|
|
1900
1629
|
threads = []
|
|
1901
|
-
|
|
1630
|
+
|
|
1902
1631
|
if !vpc_endpoints.empty?
|
|
1903
1632
|
vpc_endpoints.each { |endpoint|
|
|
1633
|
+
MU.log "Deleting VPC endpoint #{endpoint.vpc_endpoint_id}"
|
|
1634
|
+
next if noop
|
|
1904
1635
|
threads << Thread.new {
|
|
1905
|
-
MU.
|
|
1906
|
-
MU.
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if attempts % 5 == 0
|
|
1915
|
-
sleep 30
|
|
1916
|
-
begin
|
|
1917
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
|
1918
|
-
rescue Aws::EmptyStructure, NoMethodError
|
|
1919
|
-
sleep 5
|
|
1920
|
-
retry
|
|
1921
|
-
rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
|
|
1922
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
|
|
1923
|
-
end
|
|
1924
|
-
MU.log "Timed out while waiting for VPC endpoint to delete #{endpoint.vpc_endpoint_id}: #{resp}", MU::WARN if attempts > 50
|
|
1925
|
-
attempts += 1
|
|
1926
|
-
end
|
|
1927
|
-
rescue Aws::EC2::Errors::VpcEndpointIdMalformed
|
|
1928
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} was already deleted", MU::NOTICE
|
|
1929
|
-
rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
|
|
1930
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
|
|
1931
|
-
end
|
|
1932
|
-
end
|
|
1636
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
|
|
1637
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
|
1638
|
+
loop_if = Proc.new {
|
|
1639
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
|
1640
|
+
resp.state != "deleted"
|
|
1641
|
+
}
|
|
1642
|
+
MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::InvalidVpcEndpointIdNotFound, Aws::EC2::Errors::VpcEndpointIdMalformed], max: 20, wait: 10, loop_if: loop_if) { |retries, _wait|
|
|
1643
|
+
MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if retries % 5 == 0
|
|
1644
|
+
}
|
|
1933
1645
|
}
|
|
1934
1646
|
}
|
|
1935
1647
|
end
|
|
@@ -2001,56 +1713,6 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
2001
1713
|
end
|
|
2002
1714
|
private_class_method :purge_routetables
|
|
2003
1715
|
|
|
2004
|
-
# Remove all subnets associated with the currently loaded deployment.
|
|
2005
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
|
2006
|
-
# @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
|
|
2007
|
-
# @param region [String]: The cloud provider region
|
|
2008
|
-
# @return [void]
|
|
2009
|
-
def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
|
|
2010
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_subnets(
|
|
2011
|
-
filters: tagfilters
|
|
2012
|
-
)
|
|
2013
|
-
subnets = resp.data.subnets
|
|
2014
|
-
|
|
2015
|
-
return if subnets.nil? or subnets.size == 0
|
|
2016
|
-
|
|
2017
|
-
retries = 0
|
|
2018
|
-
subnets.each { |subnet|
|
|
2019
|
-
MU.log "Deleting Subnet #{subnet.subnet_id}"
|
|
2020
|
-
begin
|
|
2021
|
-
if subnet.state != "available"
|
|
2022
|
-
MU.log "Waiting for #{subnet.subnet_id} to be in a removable state...", MU::NOTICE
|
|
2023
|
-
sleep 30
|
|
2024
|
-
else
|
|
2025
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
|
|
2026
|
-
end
|
|
2027
|
-
rescue Aws::EC2::Errors::DependencyViolation => e
|
|
2028
|
-
# We're often stuck waiting for an RDS database or something else
|
|
2029
|
-
# that takes 5-ever to delete.
|
|
2030
|
-
if retries < 19
|
|
2031
|
-
loglevel = (retries > 0 and (retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
|
|
2032
|
-
MU.log "#{e.message} (retry #{retries.to_s}/20)", loglevel
|
|
2033
|
-
if loglevel == MU::NOTICE
|
|
2034
|
-
MU::Cloud::AWS::VPC.purge_interfaces(noop, [{name: "subnet-id", values: [subnet.subnet_id]}], region: region, credentials: credentials)
|
|
2035
|
-
end
|
|
2036
|
-
sleep 30
|
|
2037
|
-
retries = retries + 1
|
|
2038
|
-
retry
|
|
2039
|
-
elsif retries < 20
|
|
2040
|
-
MU.log "#{e.message} (final attempt)", MU::WARN
|
|
2041
|
-
sleep 60
|
|
2042
|
-
retries = retries + 1
|
|
2043
|
-
retry
|
|
2044
|
-
else
|
|
2045
|
-
raise e
|
|
2046
|
-
end
|
|
2047
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
|
|
2048
|
-
next
|
|
2049
|
-
end while subnet.state != "available"
|
|
2050
|
-
}
|
|
2051
|
-
end
|
|
2052
|
-
private_class_method :purge_subnets
|
|
2053
|
-
|
|
2054
1716
|
# Remove all DHCP options sets associated with the currently loaded
|
|
2055
1717
|
# deployment.
|
|
2056
1718
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
@@ -2081,6 +1743,61 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
2081
1743
|
end
|
|
2082
1744
|
private_class_method :purge_dhcpopts
|
|
2083
1745
|
|
|
1746
|
+
def self.purge_peering_connections(noop, vpc_id, region: MU.curRegion, credentials: nil)
|
|
1747
|
+
my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
|
|
1748
|
+
filters: [
|
|
1749
|
+
{
|
|
1750
|
+
name: "requester-vpc-info.vpc-id",
|
|
1751
|
+
values: [vpc_id]
|
|
1752
|
+
}
|
|
1753
|
+
]
|
|
1754
|
+
).vpc_peering_connections
|
|
1755
|
+
my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
|
|
1756
|
+
filters: [
|
|
1757
|
+
{
|
|
1758
|
+
name: "accepter-vpc-info.vpc-id",
|
|
1759
|
+
values: [vpc_id]
|
|
1760
|
+
}
|
|
1761
|
+
]
|
|
1762
|
+
).vpc_peering_connections)
|
|
1763
|
+
|
|
1764
|
+
my_peer_conns.each { |cnxn|
|
|
1765
|
+
[cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
|
|
1766
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
|
|
1767
|
+
begin
|
|
1768
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
|
|
1769
|
+
route_table_ids: [rtb_id]
|
|
1770
|
+
)
|
|
1771
|
+
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
|
|
1772
|
+
next
|
|
1773
|
+
end
|
|
1774
|
+
resp.route_tables.each { |rtb|
|
|
1775
|
+
rtb.routes.each { |route|
|
|
1776
|
+
if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
|
|
1777
|
+
MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
|
|
1778
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
|
|
1779
|
+
route_table_id: rtb_id,
|
|
1780
|
+
destination_cidr_block: route.destination_cidr_block
|
|
1781
|
+
) if !noop
|
|
1782
|
+
end
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
}
|
|
1787
|
+
MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
|
|
1788
|
+
begin
|
|
1789
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
|
|
1790
|
+
vpc_peering_connection_id: cnxn.vpc_peering_connection_id
|
|
1791
|
+
) if !noop
|
|
1792
|
+
rescue Aws::EC2::Errors::InvalidStateTransition
|
|
1793
|
+
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
|
|
1794
|
+
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
|
1795
|
+
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
|
|
1796
|
+
end
|
|
1797
|
+
}
|
|
1798
|
+
end
|
|
1799
|
+
private_class_method :purge_peering_connections
|
|
1800
|
+
|
|
2084
1801
|
# Remove all VPCs associated with the currently loaded deployment.
|
|
2085
1802
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
2086
1803
|
# @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
|
|
@@ -2088,177 +1805,40 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
|
2088
1805
|
# @return [void]
|
|
2089
1806
|
def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
|
|
2090
1807
|
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpcs(
|
|
2091
|
-
|
|
1808
|
+
filters: tagfilters
|
|
2092
1809
|
)
|
|
2093
1810
|
|
|
2094
1811
|
vpcs = resp.data.vpcs
|
|
2095
1812
|
return if vpcs.nil? or vpcs.size == 0
|
|
2096
1813
|
|
|
2097
1814
|
vpcs.each { |vpc|
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
{
|
|
2109
|
-
name: "accepter-vpc-info.vpc-id",
|
|
2110
|
-
values: [vpc.vpc_id]
|
|
2111
|
-
}
|
|
2112
|
-
]
|
|
2113
|
-
).vpc_peering_connections)
|
|
2114
|
-
my_peer_conns.each { |cnxn|
|
|
2115
|
-
|
|
2116
|
-
[cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
|
|
2117
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
|
|
2118
|
-
begin
|
|
2119
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
|
|
2120
|
-
route_table_ids: [rtb_id]
|
|
2121
|
-
)
|
|
2122
|
-
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
|
|
2123
|
-
next
|
|
2124
|
-
end
|
|
2125
|
-
resp.route_tables.each { |rtb|
|
|
2126
|
-
rtb.routes.each { |route|
|
|
2127
|
-
if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
|
|
2128
|
-
MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
|
|
2129
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
|
|
2130
|
-
route_table_id: rtb_id,
|
|
2131
|
-
destination_cidr_block: route.destination_cidr_block
|
|
2132
|
-
) if !noop
|
|
2133
|
-
end
|
|
2134
|
-
}
|
|
2135
|
-
}
|
|
2136
|
-
}
|
|
2137
|
-
}
|
|
2138
|
-
MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
|
|
2139
|
-
begin
|
|
2140
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
|
|
2141
|
-
vpc_peering_connection_id: cnxn.vpc_peering_connection_id
|
|
2142
|
-
) if !noop
|
|
2143
|
-
rescue Aws::EC2::Errors::InvalidStateTransition
|
|
2144
|
-
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
|
|
2145
|
-
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
|
2146
|
-
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
|
|
2147
|
-
end
|
|
1815
|
+
purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
|
|
1816
|
+
|
|
1817
|
+
on_retry = Proc.new {
|
|
1818
|
+
MU::Cloud.resourceClass("AWS", "FirewallRule").cleanup(
|
|
1819
|
+
noop: noop,
|
|
1820
|
+
region: region,
|
|
1821
|
+
credentials: credentials,
|
|
1822
|
+
flags: { "vpc_id" => vpc.vpc_id }
|
|
1823
|
+
)
|
|
1824
|
+
purge_gateways(noop, tagfilters, region: region, credentials: credentials)
|
|
2148
1825
|
}
|
|
2149
1826
|
|
|
2150
|
-
|
|
2151
|
-
begin
|
|
1827
|
+
MU.retrier([Aws::EC2::Errors::DependencyViolation], ignoreme: [Aws::EC2::Errors::InvalidVpcIDNotFound], max: 20, on_retry: on_retry) {
|
|
2152
1828
|
MU.log "Deleting VPC #{vpc.vpc_id}"
|
|
2153
1829
|
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc(vpc_id: vpc.vpc_id) if !noop
|
|
2154
|
-
|
|
2155
|
-
MU.log "VPC #{vpc.vpc_id} has already been deleted", MU::WARN
|
|
2156
|
-
rescue Aws::EC2::Errors::DependencyViolation => e
|
|
2157
|
-
if retries < 5
|
|
2158
|
-
MU.log "#{vpc.vpc_id} in #{region} had hidden dependencies, will try to remove them", MU::NOTICE
|
|
2159
|
-
retries += 1
|
|
2160
|
-
# fry some common rogue resources
|
|
2161
|
-
MU::Cloud::AWS::FirewallRule.cleanup(
|
|
2162
|
-
noop: noop,
|
|
2163
|
-
region: region,
|
|
2164
|
-
credentials: credentials,
|
|
2165
|
-
flags: { "vpc_id" => vpc.vpc_id }
|
|
2166
|
-
)
|
|
2167
|
-
purge_gateways(noop, tagfilters, region: region, credentials: credentials)
|
|
2168
|
-
sleep 10
|
|
2169
|
-
retry
|
|
2170
|
-
else
|
|
2171
|
-
MU.log "Failed to remove #{vpc.vpc_id} in #{region}: #{e.message}", MU::ERR
|
|
2172
|
-
next
|
|
2173
|
-
end
|
|
2174
|
-
end
|
|
1830
|
+
}
|
|
2175
1831
|
|
|
2176
1832
|
if !MU::Cloud::AWS.isGovCloud?(region)
|
|
2177
1833
|
mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
|
|
2178
1834
|
if !mu_zone.nil?
|
|
2179
|
-
MU::Cloud
|
|
1835
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
|
|
2180
1836
|
end
|
|
2181
1837
|
end
|
|
2182
1838
|
}
|
|
2183
1839
|
end
|
|
2184
1840
|
private_class_method :purge_vpcs
|
|
2185
1841
|
|
|
2186
|
-
# Subnets are almost a first-class resource. So let's kinda sorta treat
|
|
2187
|
-
# them like one. This should only be invoked on objects that already
|
|
2188
|
-
# exists in the cloud layer.
|
|
2189
|
-
class Subnet < MU::Cloud::AWS::VPC
|
|
2190
|
-
|
|
2191
|
-
attr_reader :cloud_id
|
|
2192
|
-
attr_reader :ip_block
|
|
2193
|
-
attr_reader :mu_name
|
|
2194
|
-
attr_reader :name
|
|
2195
|
-
attr_reader :az
|
|
2196
|
-
attr_reader :cloud_desc
|
|
2197
|
-
|
|
2198
|
-
# @param parent [MU::Cloud::AWS::VPC]: The parent VPC of this subnet.
|
|
2199
|
-
# @param config [Hash<String>]:
|
|
2200
|
-
def initialize(parent, config)
|
|
2201
|
-
@parent = parent
|
|
2202
|
-
@config = MU::Config.manxify(config)
|
|
2203
|
-
@cloud_id = config['cloud_id']
|
|
2204
|
-
@mu_name = config['mu_name']
|
|
2205
|
-
@name = config['name']
|
|
2206
|
-
@deploydata = config # This is a dummy for the sake of describe()
|
|
2207
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
|
|
2208
|
-
@az = resp.availability_zone
|
|
2209
|
-
@ip_block = resp.cidr_block
|
|
2210
|
-
@cloud_desc = resp # XXX this really isn't the cloud implementation's business
|
|
2211
|
-
|
|
2212
|
-
end
|
|
2213
|
-
|
|
2214
|
-
# Return the cloud identifier for the default route of this subnet.
|
|
2215
|
-
def defaultRoute
|
|
2216
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
2217
|
-
filters: [{name: "association.subnet-id", values: [@cloud_id]}]
|
|
2218
|
-
)
|
|
2219
|
-
if resp.route_tables.size == 0 # use default route table for the VPC
|
|
2220
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
2221
|
-
filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
|
|
2222
|
-
)
|
|
2223
|
-
end
|
|
2224
|
-
resp.route_tables.each { |route_table|
|
|
2225
|
-
route_table.routes.each { |route|
|
|
2226
|
-
if route.destination_cidr_block =="0.0.0.0/0" and route.state != "blackhole"
|
|
2227
|
-
return route.instance_id if !route.instance_id.nil?
|
|
2228
|
-
return route.gateway_id if !route.gateway_id.nil?
|
|
2229
|
-
return route.vpc_peering_connection_id if !route.vpc_peering_connection_id.nil?
|
|
2230
|
-
return route.network_interface_id if !route.network_interface_id.nil?
|
|
2231
|
-
end
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2234
|
-
return nil
|
|
2235
|
-
end
|
|
2236
|
-
|
|
2237
|
-
# Is this subnet privately-routable only, or public?
|
|
2238
|
-
# @return [Boolean]
|
|
2239
|
-
def private?
|
|
2240
|
-
return false if @cloud_id.nil?
|
|
2241
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
2242
|
-
filters: [{name: "association.subnet-id", values: [@cloud_id]}]
|
|
2243
|
-
)
|
|
2244
|
-
if resp.route_tables.size == 0 # use default route table for the VPC
|
|
2245
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
|
2246
|
-
filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
|
|
2247
|
-
)
|
|
2248
|
-
end
|
|
2249
|
-
resp.route_tables.each { |route_table|
|
|
2250
|
-
route_table.routes.each { |route|
|
|
2251
|
-
return false if !route.gateway_id.nil? and route.gateway_id != "local" # you can have an IgW and route it to a subset of IPs instead of 0.0.0.0/0
|
|
2252
|
-
if route.destination_cidr_block == "0.0.0.0/0"
|
|
2253
|
-
return true if !route.instance_id.nil?
|
|
2254
|
-
return true if route.nat_gateway_id
|
|
2255
|
-
end
|
|
2256
|
-
}
|
|
2257
|
-
}
|
|
2258
|
-
return true
|
|
2259
|
-
end
|
|
2260
|
-
end
|
|
2261
|
-
|
|
2262
1842
|
end #class
|
|
2263
1843
|
end #class
|
|
2264
1844
|
end
|