cloud-mu 3.1.4 → 3.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +5 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +158 -107
- data/modules/mu/adoption.rb +386 -59
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +135 -82
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +123 -21
- data/modules/mu/groomers/chef.rb +64 -11
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/master/ssl.rb +0 -1
- data/modules/mu/mommacat.rb +154 -867
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +225 -192
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +281 -64
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
- data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +14 -8
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +142 -55
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +25 -0
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +169 -93
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
- data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -120,7 +120,7 @@ module MU
|
|
120
120
|
if !@deploy.nocleanup
|
121
121
|
Thread.new {
|
122
122
|
MU.dupGlobals(parent_thread_id)
|
123
|
-
MU::Cloud
|
123
|
+
MU::Cloud.resourceClass("AWS", "Server").terminateInstance(id: member.instance_id)
|
124
124
|
}
|
125
125
|
end
|
126
126
|
end
|
@@ -193,9 +193,10 @@ module MU
|
|
193
193
|
# @return [Array<MU::Cloud::Server>]
|
194
194
|
def listNodes
|
195
195
|
nodes = []
|
196
|
-
me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id)
|
197
|
-
|
198
|
-
|
196
|
+
me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first
|
197
|
+
pp me
|
198
|
+
if me and me.instances
|
199
|
+
me.instances.each { |instance|
|
199
200
|
found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true)
|
200
201
|
nodes.concat(found)
|
201
202
|
}
|
@@ -425,6 +426,7 @@ module MU
|
|
425
426
|
# @return [OpenStruct]
|
426
427
|
def cloud_desc(use_cache: true)
|
427
428
|
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
429
|
+
return nil if !@cloud_id
|
428
430
|
@cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
|
429
431
|
auto_scaling_group_names: [@mu_name]
|
430
432
|
).auto_scaling_groups.first
|
@@ -531,14 +533,25 @@ module MU
|
|
531
533
|
if cloud_desc.vpc_zone_identifier and
|
532
534
|
!cloud_desc.vpc_zone_identifier.empty?
|
533
535
|
nets = cloud_desc.vpc_zone_identifier.split(/,/)
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
536
|
+
begin
|
537
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
|
538
|
+
bok['vpc'] = MU::Config::Ref.get(
|
539
|
+
id: resp.vpc_id,
|
540
|
+
cloud: "AWS",
|
541
|
+
credentials: @credentials,
|
542
|
+
type: "vpcs",
|
543
|
+
subnets: nets.map { |s| { "subnet_id" => s } }
|
544
|
+
)
|
545
|
+
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
546
|
+
if e.message.match(/The subnet ID '(subnet-[a-f0-9]+)' does not exist/)
|
547
|
+
nets.delete(Regexp.last_match[1])
|
548
|
+
if nets.empty?
|
549
|
+
MU.log "Autoscale Group #{@cloud_id} was configured for a VPC, but the configuration held no valid subnets", MU::WARN, details: cloud_desc.vpc_zone_identifier.split(/,/)
|
550
|
+
end
|
551
|
+
else
|
552
|
+
raise e
|
553
|
+
end
|
554
|
+
end
|
542
555
|
end
|
543
556
|
|
544
557
|
# MU.log @cloud_id, MU::NOTICE, details: cloud_desc
|
@@ -813,26 +826,7 @@ module MU
|
|
813
826
|
}
|
814
827
|
}
|
815
828
|
},
|
816
|
-
"ingress_rules" =>
|
817
|
-
"items" => {
|
818
|
-
"properties" => {
|
819
|
-
"sgs" => {
|
820
|
-
"type" => "array",
|
821
|
-
"items" => {
|
822
|
-
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
823
|
-
"type" => "string"
|
824
|
-
}
|
825
|
-
},
|
826
|
-
"lbs" => {
|
827
|
-
"type" => "array",
|
828
|
-
"items" => {
|
829
|
-
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
830
|
-
"type" => "string"
|
831
|
-
}
|
832
|
-
}
|
833
|
-
}
|
834
|
-
}
|
835
|
-
}
|
829
|
+
"ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
|
836
830
|
}
|
837
831
|
[toplevel_required, schema]
|
838
832
|
end
|
@@ -905,7 +899,7 @@ module MU
|
|
905
899
|
launch = pool["basis"]["launch_config"]
|
906
900
|
launch['iam_policies'] ||= pool['iam_policies']
|
907
901
|
|
908
|
-
launch['size'] = MU::Cloud
|
902
|
+
launch['size'] = MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(launch["size"], pool["region"])
|
909
903
|
ok = false if launch['size'].nil?
|
910
904
|
if !launch['generate_iam_role']
|
911
905
|
if !launch['iam_role'] and pool['cloud'] != "CloudFormation"
|
@@ -949,11 +943,7 @@ module MU
|
|
949
943
|
|
950
944
|
role['credentials'] = pool['credentials'] if pool['credentials']
|
951
945
|
configurator.insertKitten(role, "roles")
|
952
|
-
pool["
|
953
|
-
pool["dependencies"] << {
|
954
|
-
"type" => "role",
|
955
|
-
"name" => pool["name"]
|
956
|
-
}
|
946
|
+
MU::Config.addDependency(pool, pool['name'], "role")
|
957
947
|
end
|
958
948
|
launch["ami_id"] ||= launch["image_id"]
|
959
949
|
if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
|
@@ -967,7 +957,7 @@ module MU
|
|
967
957
|
end
|
968
958
|
end
|
969
959
|
if launch["server"] != nil
|
970
|
-
pool["
|
960
|
+
MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
|
971
961
|
# XXX I dunno, maybe toss an error if this isn't done already
|
972
962
|
# servers.each { |server|
|
973
963
|
# if server["name"] == launch["server"]
|
@@ -1073,7 +1063,7 @@ module MU
|
|
1073
1063
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
1074
1064
|
# @param region [String]: The cloud provider region
|
1075
1065
|
# @return [void]
|
1076
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
1066
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
1077
1067
|
MU.log "AWS::ServerPool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
1078
1068
|
|
1079
1069
|
filters = [{name: "key", values: ["MU-ID"]}]
|
@@ -1096,7 +1086,7 @@ module MU
|
|
1096
1086
|
if asg.key == "MU-MASTER-IP" and asg.value != MU.mu_public_ip and !ignoremaster
|
1097
1087
|
no_purge << asg.resource_id
|
1098
1088
|
end
|
1099
|
-
if asg.key == "MU-ID" and asg.value ==
|
1089
|
+
if asg.key == "MU-ID" and asg.value == deploy_id
|
1100
1090
|
maybe_purge << asg.resource_id
|
1101
1091
|
end
|
1102
1092
|
}
|
@@ -1123,7 +1113,7 @@ module MU
|
|
1123
1113
|
end
|
1124
1114
|
end
|
1125
1115
|
|
1126
|
-
# MU::Cloud
|
1116
|
+
# MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(resource_id)
|
1127
1117
|
|
1128
1118
|
# Generally there should be a launch_configuration of the same name
|
1129
1119
|
# XXX search for these independently, too?
|
@@ -1164,14 +1154,14 @@ module MU
|
|
1164
1154
|
@config['basis']['launch_config']["ami_id"] = @deploy.deployment["images"][@config['basis']['launch_config']["server"]]["image_id"]
|
1165
1155
|
MU.log "Using AMI '#{@config['basis']['launch_config']["ami_id"]}' from sibling server #{@config['basis']['launch_config']["server"]} in ServerPool #{@mu_name}"
|
1166
1156
|
elsif !@config['basis']['launch_config']["instance_id"].nil?
|
1167
|
-
@config['basis']['launch_config']["ami_id"] = MU::Cloud
|
1157
|
+
@config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
|
1168
1158
|
name: @mu_name,
|
1169
1159
|
instance_id: @config['basis']['launch_config']["instance_id"],
|
1170
1160
|
credentials: @config['credentials'],
|
1171
1161
|
region: @config['region']
|
1172
1162
|
)[@config['region']]
|
1173
1163
|
end
|
1174
|
-
MU::Cloud
|
1164
|
+
MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
|
1175
1165
|
|
1176
1166
|
oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
|
1177
1167
|
launch_configuration_names: [@mu_name]
|
@@ -1226,12 +1216,12 @@ module MU
|
|
1226
1216
|
vol.delete("encrypted")
|
1227
1217
|
end
|
1228
1218
|
end
|
1229
|
-
mapping, _cfm_mapping = MU::Cloud
|
1219
|
+
mapping, _cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
|
1230
1220
|
storage << mapping
|
1231
1221
|
}
|
1232
1222
|
end
|
1233
1223
|
|
1234
|
-
storage.concat(MU::Cloud
|
1224
|
+
storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
|
1235
1225
|
|
1236
1226
|
if @config['basis']['launch_config']['generate_iam_role']
|
1237
1227
|
role = @deploy.findLitterMate(name: @config['name'], type: "roles")
|
@@ -67,7 +67,7 @@ module MU
|
|
67
67
|
if target['vpc']["subnet_name"]
|
68
68
|
subnet_obj = vpc.getSubnet(name: target['vpc']["subnet_name"])
|
69
69
|
if subnet_obj.nil?
|
70
|
-
raise MuError, "Failed to locate subnet from #{
|
70
|
+
raise MuError, "Failed to locate subnet from #{target['vpc']["subnet_name"]} in StoragePool #{@config['name']}:#{target['name']}"
|
71
71
|
end
|
72
72
|
target['vpc']['subnet_id'] = subnet_obj.cloud_id
|
73
73
|
end
|
@@ -261,49 +261,29 @@ module MU
|
|
261
261
|
targets = {}
|
262
262
|
|
263
263
|
if @config['mount_points'] && !@config['mount_points'].empty?
|
264
|
+
mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
265
|
+
file_system_id: storage_pool.file_system_id
|
266
|
+
).mount_targets
|
267
|
+
|
264
268
|
@config['mount_points'].each { |mp|
|
265
269
|
subnet = nil
|
266
270
|
dependencies
|
267
|
-
mp_vpc =
|
268
|
-
@deploy.findLitterMate(type: "vpc", name: mp['vpc']['vpc_name'], credentials: @config['credentials'])
|
269
|
-
elsif mp['vpc']
|
270
|
-
MU::MommaCat.findStray(
|
271
|
-
@config['cloud'],
|
272
|
-
"vpcs",
|
273
|
-
deploy_id: mp['vpc']["deploy_id"],
|
274
|
-
credentials: @config['credentials'],
|
275
|
-
mu_name: mp['vpc']["mu_name"],
|
276
|
-
cloud_id: mp['vpc']['vpc_id'],
|
277
|
-
region: @config['region'],
|
278
|
-
dummy_ok: false
|
279
|
-
).first
|
280
|
-
# XXX non-sibling, findStray version
|
281
|
-
end
|
271
|
+
mp_vpc = MU::Config::Ref.get(mp['vpc']).kitten
|
282
272
|
|
283
|
-
mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
284
|
-
file_system_id: storage_pool.file_system_id
|
285
|
-
).mount_targets
|
286
273
|
|
287
|
-
|
288
|
-
|
289
|
-
|
274
|
+
subnet_obj = mp_vpc.subnets.select { |s|
|
275
|
+
s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
|
276
|
+
}.first
|
290
277
|
mount_target = nil
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
}
|
299
|
-
break if mount_target
|
278
|
+
mount_targets.each { |t|
|
279
|
+
subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
|
280
|
+
if subnet_cidr_obj.contains(NetAddr::IPv4.parse(t.ip_address))
|
281
|
+
mount_target = t
|
282
|
+
subnet = subnet_obj.cloud_desc
|
283
|
+
break
|
284
|
+
end
|
300
285
|
}
|
301
286
|
|
302
|
-
# mount_target = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
|
303
|
-
# mount_target_id: mp["cloud_id"]
|
304
|
-
# ).mount_targets.first
|
305
|
-
|
306
|
-
|
307
287
|
targets[mp["name"]] = {
|
308
288
|
"owner_id" => mount_target.owner_id,
|
309
289
|
"cloud_id" => mount_target.mount_target_id,
|
@@ -353,7 +333,7 @@ module MU
|
|
353
333
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
354
334
|
# @param region [String]: The cloud provider region in which to operate
|
355
335
|
# @return [void]
|
356
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
336
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
357
337
|
MU.log "AWS::StoragePool.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
358
338
|
|
359
339
|
supported_regions = %w{us-west-2 us-east-1 eu-west-1}
|
@@ -378,7 +358,7 @@ module MU
|
|
378
358
|
found_muid = false
|
379
359
|
found_master = false
|
380
360
|
tags.each { |tag|
|
381
|
-
found_muid = true if tag.key == "MU-ID" && tag.value ==
|
361
|
+
found_muid = true if tag.key == "MU-ID" && tag.value == deploy_id
|
382
362
|
found_master = true if tag.key == "MU-MASTER-IP" && tag.value == MU.mu_public_ip
|
383
363
|
}
|
384
364
|
next if !found_muid
|
@@ -493,6 +473,9 @@ module MU
|
|
493
473
|
|
494
474
|
if pool['mount_points'] && !pool['mount_points'].empty?
|
495
475
|
pool['mount_points'].each{ |mp|
|
476
|
+
if mp['vpc'] and mp['vpc']['name']
|
477
|
+
MU::Config.addDependency(pool, mp['vpc']['name'], "vpc")
|
478
|
+
end
|
496
479
|
if mp['ingress_rules']
|
497
480
|
fwname = "storage-#{mp['name']}"
|
498
481
|
acl = {
|
@@ -109,7 +109,7 @@ module MU
|
|
109
109
|
# Create these if necessary, then append them to the list of
|
110
110
|
# attachable_policies
|
111
111
|
if @config['raw_policies']
|
112
|
-
pol_arns = MU::Cloud
|
112
|
+
pol_arns = MU::Cloud.resourceClass("AWS", "Role").manageRawPolicies(
|
113
113
|
@config['raw_policies'],
|
114
114
|
basename: @deploy.getResourceName(@config['name']),
|
115
115
|
credentials: @credentials
|
@@ -135,7 +135,7 @@ module MU
|
|
135
135
|
attached_policies.each { |a|
|
136
136
|
if !configured_policies.include?(a.policy_arn)
|
137
137
|
MU.log "Removing IAM policy #{a.policy_arn} from user #{@mu_name}", MU::NOTICE
|
138
|
-
MU::Cloud
|
138
|
+
MU::Cloud.resourceClass("AWS", "Role").purgePolicy(a.policy_arn, @credentials)
|
139
139
|
else
|
140
140
|
configured_policies.delete(a.policy_arn)
|
141
141
|
end
|
@@ -151,7 +151,7 @@ module MU
|
|
151
151
|
end
|
152
152
|
|
153
153
|
if @config['inline_policies']
|
154
|
-
docs = MU::Cloud
|
154
|
+
docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['inline_policies'], deploy_obj: @deploy)
|
155
155
|
docs.each { |doc|
|
156
156
|
MU.log "Putting user policy #{doc.keys.first} to user #{@cloud_id} "
|
157
157
|
MU::Cloud::AWS.iam(credentials: @credentials).put_user_policy(
|
@@ -190,16 +190,16 @@ module MU
|
|
190
190
|
# @param noop [Boolean]: If true, will only print what would be done
|
191
191
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
192
192
|
# @return [void]
|
193
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
193
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
194
194
|
MU.log "AWS::User.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
195
195
|
|
196
196
|
# XXX this doesn't belong here; maybe under roles, maybe as its own stupid first-class resource
|
197
197
|
resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
|
198
|
-
path_prefix: "/"+
|
198
|
+
path_prefix: "/"+deploy_id+"/"
|
199
199
|
)
|
200
200
|
if resp and resp.policies
|
201
201
|
resp.policies.each { |policy|
|
202
|
-
MU.log "Deleting policy /#{
|
202
|
+
MU.log "Deleting policy /#{deploy_id}/#{policy.policy_name}"
|
203
203
|
if !noop
|
204
204
|
attachments = begin
|
205
205
|
MU::Cloud::AWS.iam(credentials: credentials).list_entities_for_policy(
|
@@ -277,7 +277,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
277
277
|
has_ourdeploy = false
|
278
278
|
has_ourmaster = false
|
279
279
|
tags.each { |tag|
|
280
|
-
if tag.key == "MU-ID" and tag.value ==
|
280
|
+
if tag.key == "MU-ID" and tag.value == deploy_id
|
281
281
|
has_ourdeploy = true
|
282
282
|
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
283
283
|
has_ourmaster = true
|
@@ -431,7 +431,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
431
431
|
resp.policy_names.each { |pol_name|
|
432
432
|
pol = MU::Cloud::AWS.iam(credentials: @credentials).get_user_policy(user_name: @cloud_id, policy_name: pol_name)
|
433
433
|
doc = JSON.parse(URI.decode(pol.policy_document))
|
434
|
-
bok["inline_policies"] = MU::Cloud
|
434
|
+
bok["inline_policies"] = MU::Cloud.resourceClass("AWS", "Role").doc2MuPolicies(pol.policy_name, doc, bok["inline_policies"])
|
435
435
|
}
|
436
436
|
end
|
437
437
|
|
@@ -465,7 +465,7 @@ MU.log e.inspect, MU::ERR, details: policy
|
|
465
465
|
def self.schema(_config)
|
466
466
|
toplevel_required = []
|
467
467
|
polschema = MU::Config::Role.schema["properties"]["policies"]
|
468
|
-
polschema.deep_merge!(MU::Cloud
|
468
|
+
polschema.deep_merge!(MU::Cloud.resourceClass("AWS", "Role").condition_schema)
|
469
469
|
|
470
470
|
schema = {
|
471
471
|
"inline_policies" => polschema,
|
@@ -517,7 +517,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
517
517
|
# If we're attaching some managed policies, make sure all of the ones
|
518
518
|
# that should already exist do indeed exist
|
519
519
|
if user['attachable_policies']
|
520
|
-
ok = false if !MU::Cloud
|
520
|
+
ok = false if !MU::Cloud.resourceClass("AWS", "Role").validateAttachablePolicies(
|
521
521
|
user['attachable_policies'],
|
522
522
|
credentials: user['credentials'],
|
523
523
|
region: user['region']
|
@@ -530,7 +530,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
530
530
|
if configurator.haveLitterMate?(group, "groups")
|
531
531
|
need_dependency = true
|
532
532
|
else
|
533
|
-
found = MU::Cloud
|
533
|
+
found = MU::Cloud.resourceClass("AWS", "Group").find(cloud_id: group)
|
534
534
|
if found.nil? or found.empty? or (configurator.updating and
|
535
535
|
found.values.first.group.path == "/"+configurator.updating+"/")
|
536
536
|
groupdesc = {
|
@@ -542,11 +542,7 @@ style long name, like +IAMTESTS-DEV-2018112815-IS-USER-FOO+"
|
|
542
542
|
end
|
543
543
|
|
544
544
|
if need_dependency
|
545
|
-
user
|
546
|
-
user["dependencies"] << {
|
547
|
-
"type" => "group",
|
548
|
-
"name" => group
|
549
|
-
}
|
545
|
+
MU::Config.addDependency(user, group, "group")
|
550
546
|
end
|
551
547
|
}
|
552
548
|
end
|
File without changes
|
@@ -42,7 +42,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
42
42
|
<% if !$mu.skipApplyUpdates %>
|
43
43
|
set +e
|
44
44
|
if [ ! -f /.mu-installer-ran-updates ];then
|
45
|
-
|
45
|
+
echo "Applying package updates" > /etc/nologin
|
46
46
|
apt-get --fix-missing -y upgrade
|
47
47
|
touch /.mu-installer-ran-updates
|
48
48
|
if [ $? -eq 0 ]
|
@@ -58,7 +58,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
58
58
|
else
|
59
59
|
echo "FAILED PACKAGE UPDATE" >&2
|
60
60
|
fi
|
61
|
-
|
61
|
+
rm -f /etc/nologin
|
62
62
|
fi
|
63
63
|
<% end %>
|
64
64
|
elif [ -x /usr/bin/yum ];then
|
@@ -94,7 +94,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
94
94
|
<% if !$mu.skipApplyUpdates %>
|
95
95
|
set +e
|
96
96
|
if [ ! -f /.mu-installer-ran-updates ];then
|
97
|
-
|
97
|
+
echo "Applying package updates" > /etc/nologin
|
98
98
|
kernel_update=`yum list updates | grep kernel`
|
99
99
|
yum -y update
|
100
100
|
touch /.mu-installer-ran-updates
|
@@ -108,7 +108,7 @@ if ping -c 5 8.8.8.8 > /dev/null; then
|
|
108
108
|
else
|
109
109
|
echo "FAILED PACKAGE UPDATE" >&2
|
110
110
|
fi
|
111
|
-
|
111
|
+
rm -f /etc/nologin
|
112
112
|
fi
|
113
113
|
<% end %>
|
114
114
|
fi
|
@@ -116,6 +116,7 @@ else
|
|
116
116
|
/bin/logger "***** Unable to verify internet connectivity, skipping package updates from userdata"
|
117
117
|
touch /.mu-installer-ran-updates
|
118
118
|
fi
|
119
|
+
rm -f /etc/nologin
|
119
120
|
|
120
121
|
AWSCLI='command -v aws'
|
121
122
|
PIP='command -v pip'
|
@@ -23,7 +23,7 @@ function log
|
|
23
23
|
}
|
24
24
|
|
25
25
|
function fetchSecret([string]$file){
|
26
|
-
log "
|
26
|
+
log "aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file"
|
27
27
|
aws.cmd --region $region s3 cp s3://<%= $mu.adminBucketName %>/$file $tmp/$file
|
28
28
|
}
|
29
29
|
|
@@ -245,6 +245,7 @@ if($ingroup -ne $admin_username){
|
|
245
245
|
net localgroup WinRMRemoteWMIUsers__ /add $admin_username
|
246
246
|
}
|
247
247
|
|
248
|
+
importCert "$myname-winrm.crt" "root"
|
248
249
|
$winrmcert = importCert "$myname-winrm.crt" "TrustedPeople"
|
249
250
|
Set-Item -Path WSMan:\localhost\Service\Auth\Certificate -Value $true
|
250
251
|
Set-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" -Name LocalAccountTokenFilterPolicy -Value 1
|
@@ -18,6 +18,7 @@ module MU
|
|
18
18
|
|
19
19
|
# Creation of Virtual Private Clouds and associated artifacts (routes, subnets, etc).
|
20
20
|
class VPC < MU::Cloud::VPC
|
21
|
+
require 'mu/providers/aws/vpc_subnet'
|
21
22
|
|
22
23
|
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
|
23
24
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
@@ -35,78 +36,40 @@ module MU
|
|
35
36
|
def create
|
36
37
|
MU.log "Creating VPC #{@mu_name}", details: @config
|
37
38
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc(cidr_block: @config['ip_block']).vpc
|
38
|
-
|
39
|
+
@cloud_id = resp.vpc_id
|
40
|
+
@config['vpc_id'] = @cloud_id
|
39
41
|
|
40
|
-
|
41
|
-
MU::Cloud::AWS.createTag(vpc_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
42
|
-
|
43
|
-
if @config['tags']
|
44
|
-
@config['tags'].each { |tag|
|
45
|
-
MU::Cloud::AWS.createTag(vpc_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
46
|
-
}
|
47
|
-
end
|
48
|
-
|
49
|
-
if @config['optional_tags']
|
50
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
51
|
-
MU::Cloud::AWS.createTag(vpc_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
52
|
-
}
|
53
|
-
end
|
42
|
+
tag_me
|
54
43
|
|
55
44
|
if resp.state != "available"
|
56
45
|
begin
|
57
|
-
MU.log "Waiting for VPC #{@mu_name} (#{
|
46
|
+
MU.log "Waiting for VPC #{@mu_name} (#{@cloud_id}) to be available", MU::NOTICE
|
58
47
|
sleep 5
|
59
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [
|
48
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpcs(vpc_ids: [@cloud_id]).vpcs.first
|
60
49
|
end while resp.state != "available"
|
61
50
|
# There's a default route table that comes with. Let's tag it.
|
62
51
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
63
52
|
filters: [
|
64
53
|
{
|
65
54
|
name: "vpc-id",
|
66
|
-
values: [
|
55
|
+
values: [@cloud_id]
|
67
56
|
}
|
68
57
|
]
|
69
58
|
)
|
70
59
|
resp.route_tables.each { |rtb|
|
71
|
-
|
72
|
-
if @config['tags']
|
73
|
-
@config['tags'].each { |tag|
|
74
|
-
MU::Cloud::AWS.createTag(rtb.route_table_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
75
|
-
}
|
76
|
-
end
|
77
|
-
|
78
|
-
MU::Cloud::AWS.createStandardTags(rtb.route_table_id, region: @config['region'], credentials: @config['credentials'])
|
79
|
-
|
80
|
-
if @config['optional_tags']
|
81
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
82
|
-
MU::Cloud::AWS.createTag(rtb.route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
83
|
-
}
|
84
|
-
end
|
60
|
+
tag_me(rtb.route_table_id, @mu_name+"-#DEFAULTPRIV")
|
85
61
|
}
|
86
62
|
end
|
87
|
-
@config['vpc_id'] = vpc_id
|
88
|
-
@cloud_id = vpc_id
|
89
63
|
|
90
64
|
if @config['create_internet_gateway']
|
91
65
|
MU.log "Creating Internet Gateway #{@mu_name}"
|
92
66
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_internet_gateway
|
93
67
|
internet_gateway_id = resp.internet_gateway.internet_gateway_id
|
94
68
|
sleep 5
|
95
|
-
MU::Cloud::AWS.createStandardTags(internet_gateway_id, region: @config['region'], credentials: @config['credentials'])
|
96
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
97
|
-
if @config['tags']
|
98
|
-
@config['tags'].each { |tag|
|
99
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
100
|
-
}
|
101
|
-
end
|
102
69
|
|
103
|
-
|
104
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
105
|
-
MU::Cloud::AWS.createTag(internet_gateway_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
106
|
-
}
|
107
|
-
end
|
70
|
+
tag_me(internet_gateway_id)
|
108
71
|
|
109
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id:
|
72
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_internet_gateway(vpc_id: @cloud_id, internet_gateway_id: internet_gateway_id)
|
110
73
|
@config['internet_gateway_id'] = internet_gateway_id
|
111
74
|
end
|
112
75
|
|
@@ -165,236 +128,35 @@ module MU
|
|
165
128
|
)
|
166
129
|
end
|
167
130
|
|
168
|
-
nat_gateways =
|
169
|
-
if !@config['subnets'].nil?
|
170
|
-
allocation_ids = []
|
171
|
-
subnetthreads = Array.new
|
172
|
-
parent_thread_id = Thread.current.object_id
|
173
|
-
azs = []
|
174
|
-
@config['subnets'].each { |subnet|
|
175
|
-
subnet_name = @config['name']+"-"+subnet['name']
|
176
|
-
MU.log "Creating Subnet #{subnet_name} (#{subnet['ip_block']})", details: subnet
|
177
|
-
azs = MU::Cloud::AWS.listAZs(region: @config['region'], credentials: @config['credentials']) if azs.size == 0
|
178
|
-
if !subnet['availability_zone'].nil?
|
179
|
-
az = subnet['availability_zone']
|
180
|
-
else
|
181
|
-
az = azs.pop
|
182
|
-
end
|
183
|
-
|
184
|
-
subnetthreads << Thread.new {
|
185
|
-
MU.dupGlobals(parent_thread_id)
|
186
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_subnet(
|
187
|
-
vpc_id: vpc_id,
|
188
|
-
cidr_block: subnet['ip_block'],
|
189
|
-
availability_zone: az
|
190
|
-
).subnet
|
191
|
-
subnet_id = subnet['subnet_id'] = resp.subnet_id
|
192
|
-
MU::Cloud::AWS.createStandardTags(subnet_id, region: @config['region'], credentials: @config['credentials'])
|
193
|
-
MU::Cloud::AWS.createTag(subnet_id, "Name", @mu_name+"-"+subnet['name'], region: @config['region'], credentials: @config['credentials'])
|
194
|
-
if @config['tags']
|
195
|
-
@config['tags'].each { |tag|
|
196
|
-
MU::Cloud::AWS.createTag(subnet_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
197
|
-
}
|
198
|
-
end
|
199
|
-
|
200
|
-
if @config['optional_tags']
|
201
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
202
|
-
MU::Cloud::AWS.createTag(subnet_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
203
|
-
}
|
204
|
-
end
|
205
|
-
|
206
|
-
retries = 0
|
207
|
-
begin
|
208
|
-
if resp.state != "available"
|
209
|
-
begin
|
210
|
-
MU.log "Waiting for Subnet #{subnet_name} (#{subnet_id}) to be available", MU::NOTICE if retries > 0 and (retries % 3) == 0
|
211
|
-
sleep 5
|
212
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
213
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
214
|
-
sleep 10
|
215
|
-
retry
|
216
|
-
end while resp.state != "available"
|
217
|
-
end
|
218
|
-
rescue NoMethodError => e
|
219
|
-
if retries <= 3
|
220
|
-
MU.log "Got bogus Aws::EmptyResponse error on #{subnet_id} (retries used: #{retries}/3)", MU::WARN
|
221
|
-
retries = retries + 1
|
222
|
-
sleep 5
|
223
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
224
|
-
retry
|
225
|
-
else
|
226
|
-
raise e
|
227
|
-
end
|
228
|
-
end
|
229
|
-
|
230
|
-
if !subnet['route_table'].nil?
|
231
|
-
routes = {}
|
232
|
-
@config['route_tables'].each { |tbl|
|
233
|
-
routes[tbl['name']] = tbl
|
234
|
-
}
|
235
|
-
if routes.nil? or routes[subnet['route_table']].nil?
|
236
|
-
MU.log "Subnet #{subnet_name} references non-existent route #{subnet['route_table']}", MU::ERR, details: @deploy.deployment['vpcs']
|
237
|
-
raise MuError, "deploy failure"
|
238
|
-
end
|
239
|
-
MU.log "Associating Route Table '#{subnet['route_table']}' (#{routes[subnet['route_table']]['route_table_id']}) with #{subnet_name}"
|
240
|
-
retries = 0
|
241
|
-
begin
|
242
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_route_table(
|
243
|
-
route_table_id: routes[subnet['route_table']]['route_table_id'],
|
244
|
-
subnet_id: subnet_id
|
245
|
-
)
|
246
|
-
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound => e
|
247
|
-
retries = retries + 1
|
248
|
-
if retries < 10
|
249
|
-
sleep 10
|
250
|
-
retry
|
251
|
-
else
|
252
|
-
raise MuError, e.inspect
|
253
|
-
end
|
254
|
-
end
|
255
|
-
end
|
256
|
-
retries = 0
|
257
|
-
begin
|
258
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [subnet_id]).subnets.first
|
259
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
260
|
-
if retries < 10
|
261
|
-
MU.log "Got #{e.inspect}, waiting and retrying", MU::WARN
|
262
|
-
sleep 10
|
263
|
-
retries = retries + 1
|
264
|
-
retry
|
265
|
-
end
|
266
|
-
raise MuError, e.inspect, e.backtrace
|
267
|
-
end
|
268
|
-
|
269
|
-
if subnet['is_public'] && subnet['create_nat_gateway']
|
270
|
-
MU::MommaCat.lock("nat-gateway-eipalloc")
|
271
|
-
filters = [{name: "domain", values: ["vpc"]}]
|
272
|
-
eips = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_addresses(filters: filters).addresses
|
273
|
-
allocation_id = nil
|
274
|
-
eips.each { |eip|
|
275
|
-
next if !eip.association_id.nil? and !eip.association_id.empty?
|
276
|
-
if (eip.private_ip_address.nil? || eip.private_ip_address.empty?) and MU::MommaCat.lock(eip.allocation_id, true, true)
|
277
|
-
if !allocation_ids.include?(eip.allocation_id)
|
278
|
-
allocation_id = eip.allocation_id
|
279
|
-
break
|
280
|
-
end
|
281
|
-
end
|
282
|
-
}
|
283
|
-
|
284
|
-
if allocation_id.nil?
|
285
|
-
allocation_id = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).allocate_address(domain: "vpc").allocation_id
|
286
|
-
MU::MommaCat.lock(allocation_id, false, true)
|
287
|
-
end
|
131
|
+
nat_gateways = create_subnets
|
288
132
|
|
289
|
-
|
290
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_nat_gateway(
|
291
|
-
subnet_id: subnet['subnet_id'],
|
292
|
-
allocation_id: allocation_id,
|
293
|
-
).nat_gateway
|
294
|
-
|
295
|
-
nat_gateway_id = resp.nat_gateway_id
|
296
|
-
attempts = 0
|
297
|
-
MU::MommaCat.unlock("nat-gateway-eipalloc")
|
298
|
-
while resp.class.name != "Aws::EC2::Types::NatGateway" or resp.state == "pending"
|
299
|
-
MU.log "Waiting for nat gateway #{nat_gateway_id} () to become available (EIP allocation: #{allocation_id})" if attempts % 5 == 0
|
300
|
-
sleep 30
|
301
|
-
begin
|
302
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_nat_gateways(nat_gateway_ids: [nat_gateway_id]).nat_gateways.first
|
303
|
-
rescue Aws::EmptyStructure, NoMethodError
|
304
|
-
sleep 5
|
305
|
-
retry
|
306
|
-
end
|
307
|
-
if attempts > 30
|
308
|
-
MU::MommaCat.unlock(allocation_id, true)
|
309
|
-
raise MuError, "Timed out while waiting for NAT Gateway #{nat_gateway_id}: #{resp}"
|
310
|
-
end
|
311
|
-
attempts += 1
|
312
|
-
end
|
313
|
-
MU::MommaCat.unlock(allocation_id, true)
|
314
|
-
|
315
|
-
raise MuError, "NAT Gateway failed #{nat_gateway_id}: #{resp}" if resp.state == "failed"
|
316
|
-
nat_gateways << {'id' => nat_gateway_id, 'availability_zone' => subnet['availability_zone']}
|
317
|
-
@tags.each_pair { |k, v|
|
318
|
-
MU::Cloud::AWS.createTag(nat_gateway_id, k, v, region: @config['region'], credentials: @config['credentials'])
|
319
|
-
}
|
320
|
-
end
|
321
|
-
|
322
|
-
if subnet.has_key?("map_public_ips")
|
323
|
-
retries = 0
|
324
|
-
begin
|
325
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_subnet_attribute(
|
326
|
-
subnet_id: subnet_id,
|
327
|
-
map_public_ip_on_launch: {
|
328
|
-
value: subnet['map_public_ips'],
|
329
|
-
}
|
330
|
-
)
|
331
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound => e
|
332
|
-
if retries < 10
|
333
|
-
MU.log "Got #{e.inspect} while trying to enable map_public_ips on subnet, waiting and retrying", MU::WARN
|
334
|
-
sleep 10
|
335
|
-
retries += 1
|
336
|
-
retry
|
337
|
-
end
|
338
|
-
raise MuError, "Got #{e.inspect}, #{e.backtrace} while trying to enable map_public_ips on subnet"
|
339
|
-
end
|
340
|
-
end
|
341
|
-
|
342
|
-
if subnet["enable_traffic_logging"]
|
343
|
-
loggroup = @deploy.findLitterMate(name: @config['name']+"loggroup", type: "logs")
|
344
|
-
logrole = @deploy.findLitterMate(name: @config['name']+"logrole", type: "roles")
|
345
|
-
MU.log "Enabling traffic logging on Subnet #{subnet_name} in VPC #{@mu_name} to log group #{loggroup.mu_name}"
|
346
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_flow_logs(
|
347
|
-
resource_ids: [subnet_id],
|
348
|
-
resource_type: "Subnet",
|
349
|
-
traffic_type: subnet["traffic_type_to_log"],
|
350
|
-
log_group_name: loggroup.mu_name,
|
351
|
-
deliver_logs_permission_arn: logrole.cloudobj.arn
|
352
|
-
)
|
353
|
-
end
|
354
|
-
}
|
355
|
-
}
|
356
|
-
|
357
|
-
subnetthreads.each { |t|
|
358
|
-
t.join
|
359
|
-
}
|
360
|
-
|
361
|
-
notify
|
362
|
-
end
|
133
|
+
notify
|
363
134
|
|
364
135
|
if !nat_gateways.empty?
|
365
136
|
nat_gateways.each { |gateway|
|
366
137
|
@config['subnets'].each { |subnet|
|
367
|
-
if subnet['is_public']
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
else
|
388
|
-
raise e
|
389
|
-
end
|
390
|
-
rescue Aws::EC2::Errors::RouteAlreadyExists => e
|
391
|
-
MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
|
392
|
-
end
|
393
|
-
end
|
394
|
-
}
|
395
|
-
end
|
138
|
+
next if subnet['is_public'] != false or subnet['availability_zone'] != gateway['availability_zone']
|
139
|
+
|
140
|
+
@config['route_tables'].each { |rtb|
|
141
|
+
next if rtb['name'] != subnet['route_table']
|
142
|
+
rtb['routes'].each { |route|
|
143
|
+
next if route['gateway'] != '#NAT'
|
144
|
+
route_config = {
|
145
|
+
:route_table_id => rtb['route_table_id'],
|
146
|
+
:destination_cidr_block => route['destination_network'],
|
147
|
+
:nat_gateway_id => gateway['id']
|
148
|
+
}
|
149
|
+
|
150
|
+
MU.log "Creating route for #{route['destination_network']} through NAT gatway #{gateway['id']}", details: route_config
|
151
|
+
MU.retrier([Aws::EC2::Errors::InvalidNatGatewayIDNotFound], wait: 10, max: 5) {
|
152
|
+
begin
|
153
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(route_config)
|
154
|
+
rescue Aws::EC2::Errors::RouteAlreadyExists
|
155
|
+
MU.log "Attempt to create duplicate route to #{route['destination_network']} for #{gateway['id']} in #{rtb['route_table_id']}", MU::WARN
|
156
|
+
end
|
157
|
+
}
|
396
158
|
}
|
397
|
-
|
159
|
+
}
|
398
160
|
}
|
399
161
|
}
|
400
162
|
end
|
@@ -402,14 +164,14 @@ module MU
|
|
402
164
|
if @config['enable_dns_support']
|
403
165
|
MU.log "Enabling DNS support in #{@mu_name}"
|
404
166
|
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
|
405
|
-
vpc_id:
|
167
|
+
vpc_id: @cloud_id,
|
406
168
|
enable_dns_support: {value: @config['enable_dns_support']}
|
407
169
|
)
|
408
170
|
end
|
409
171
|
if @config['enable_dns_hostnames']
|
410
172
|
MU.log "Enabling DNS hostnames in #{@mu_name}"
|
411
173
|
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).modify_vpc_attribute(
|
412
|
-
vpc_id:
|
174
|
+
vpc_id: @cloud_id,
|
413
175
|
enable_dns_hostnames: {value: @config['enable_dns_hostnames']}
|
414
176
|
)
|
415
177
|
end
|
@@ -438,29 +200,16 @@ module MU
|
|
438
200
|
dhcp_configurations: dhcpopts
|
439
201
|
)
|
440
202
|
dhcpopt_id = resp.dhcp_options.dhcp_options_id
|
441
|
-
|
442
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
|
443
|
-
|
444
|
-
if @config['tags']
|
445
|
-
@config['tags'].each { |tag|
|
446
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
447
|
-
}
|
448
|
-
end
|
449
|
-
|
450
|
-
if @config['optional_tags']
|
451
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
452
|
-
MU::Cloud::AWS.createTag(dhcpopt_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
453
|
-
}
|
454
|
-
end
|
203
|
+
tag_me(dhcpopt_id)
|
455
204
|
|
456
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id:
|
205
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).associate_dhcp_options(dhcp_options_id: dhcpopt_id, vpc_id: @cloud_id)
|
457
206
|
end
|
458
207
|
notify
|
459
208
|
|
460
209
|
if !MU::Cloud::AWS.isGovCloud?(@config['region'])
|
461
210
|
mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", credentials: @config['credentials']).values.first
|
462
211
|
if !mu_zone.nil?
|
463
|
-
MU::Cloud
|
212
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: @cloud_id, region: @config['region'], credentials: @config['credentials'])
|
464
213
|
end
|
465
214
|
end
|
466
215
|
loadSubnets
|
@@ -471,9 +220,6 @@ module MU
|
|
471
220
|
# Canonical Amazon Resource Number for this resource
|
472
221
|
# @return [String]
|
473
222
|
def arn
|
474
|
-
puts @config['region']
|
475
|
-
puts MU::Cloud::AWS.credToAcct(@config['credentials'])
|
476
|
-
puts @cloud_id
|
477
223
|
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":vpc/"+@cloud_id
|
478
224
|
end
|
479
225
|
|
@@ -490,202 +236,7 @@ module MU
|
|
490
236
|
# Generate peering connections
|
491
237
|
if !@config['peers'].nil? and @config['peers'].size > 0
|
492
238
|
@config['peers'].each { |peer|
|
493
|
-
|
494
|
-
peer_id = nil
|
495
|
-
peer['name'] ||= peer['vpc_name']
|
496
|
-
peer['id'] ||= peer['vpc_id']
|
497
|
-
|
498
|
-
# If we know this to be a sibling VPC elsewhere in our stack,
|
499
|
-
# go fetch it, and fix it if we've been misconfigured with a
|
500
|
-
# duplicate peering connection
|
501
|
-
if peer['vpc']['name'] and !peer['account']
|
502
|
-
peer_obj = @deploy.findLitterMate(name: peer['vpc']['name'], type: "vpcs")
|
503
|
-
if peer_obj
|
504
|
-
if peer_obj.config['peers']
|
505
|
-
skipme = false
|
506
|
-
peer_obj.config['peers'].each { |peerpeer|
|
507
|
-
if peerpeer['vpc']['name'] == @config['name'] and
|
508
|
-
(peer['vpc']['name'] <=> @config['name']) == -1
|
509
|
-
skipme = true
|
510
|
-
MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
|
511
|
-
# XXX and if deploy_id matches or is unset
|
512
|
-
end
|
513
|
-
}
|
514
|
-
end
|
515
|
-
next if skipme
|
516
|
-
peer['account'] = MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
517
|
-
peer['vpc']['id'] = peer_obj.cloud_id
|
518
|
-
peer['vpc']['region'] ||= peer_obj.config['region']
|
519
|
-
end
|
520
|
-
end
|
521
|
-
|
522
|
-
# If we still don't know our peer's vpc identifier, go fishing
|
523
|
-
if !peer_obj
|
524
|
-
tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
|
525
|
-
if peer['vpc']['deploy_id'].nil? and peer['vpc']['id'].nil? and tag_key.nil?
|
526
|
-
peer['vpc']['deploy_id'] = @deploy.deploy_id
|
527
|
-
end
|
528
|
-
peer_obj = MU::MommaCat.findStray(
|
529
|
-
"AWS",
|
530
|
-
"vpcs",
|
531
|
-
deploy_id: peer['vpc']['deploy_id'],
|
532
|
-
cloud_id: peer['vpc']['id'],
|
533
|
-
# XXX we need a credentials argument here... maybe
|
534
|
-
name: peer['vpc']['name'],
|
535
|
-
tag_key: tag_key,
|
536
|
-
tag_value: tag_value,
|
537
|
-
dummy_ok: true,
|
538
|
-
region: peer['vpc']['region']
|
539
|
-
)
|
540
|
-
MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
|
541
|
-
raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
|
542
|
-
peer_obj = peer_obj.first
|
543
|
-
peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
544
|
-
peer['vpc']['id'] ||= peer_obj.cloud_id
|
545
|
-
peer['vpc']['region'] ||= peer_obj.config['region']
|
546
|
-
end
|
547
|
-
|
548
|
-
peer_id = peer['vpc']['id']
|
549
|
-
peer['account'] ||= MU::Cloud::AWS.account_number
|
550
|
-
|
551
|
-
# See if the peering connection exists before we bother
|
552
|
-
# creating it.
|
553
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
554
|
-
filters: [
|
555
|
-
{
|
556
|
-
name: "requester-vpc-info.vpc-id",
|
557
|
-
values: [@cloud_id]
|
558
|
-
},
|
559
|
-
{
|
560
|
-
name: "accepter-vpc-info.vpc-id",
|
561
|
-
values: [peer_id.to_s]
|
562
|
-
}
|
563
|
-
]
|
564
|
-
)
|
565
|
-
|
566
|
-
peering_id = if !resp or !resp.vpc_peering_connections or
|
567
|
-
resp.vpc_peering_connections.empty?
|
568
|
-
|
569
|
-
MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
|
570
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
|
571
|
-
vpc_id: @cloud_id,
|
572
|
-
peer_vpc_id: peer_id,
|
573
|
-
peer_owner_id: peer['account'],
|
574
|
-
peer_region: peer['vpc']['region']
|
575
|
-
)
|
576
|
-
resp.vpc_peering_connection.vpc_peering_connection_id
|
577
|
-
else
|
578
|
-
resp.vpc_peering_connections.first.vpc_peering_connection_id
|
579
|
-
end
|
580
|
-
|
581
|
-
peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer['vpc']['id'])
|
582
|
-
|
583
|
-
MU::Cloud::AWS.createStandardTags(peering_id, region: @config['region'], credentials: @config['credentials'])
|
584
|
-
MU::Cloud::AWS.createTag(peering_id, "Name", peering_name, region: @config['region'], credentials: @config['credentials'])
|
585
|
-
|
586
|
-
if @config['optional_tags']
|
587
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
588
|
-
MU::Cloud::AWS.createTag(peering_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
589
|
-
}
|
590
|
-
end
|
591
|
-
|
592
|
-
if @config['tags']
|
593
|
-
@config['tags'].each { |tag|
|
594
|
-
MU::Cloud::AWS.createTag(peering_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
595
|
-
}
|
596
|
-
end
|
597
|
-
|
598
|
-
# Create routes to our new friend.
|
599
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
|
600
|
-
my_route_config = {
|
601
|
-
:route_table_id => rtb_id,
|
602
|
-
:destination_cidr_block => peer_obj.cloud_desc.cidr_block,
|
603
|
-
:vpc_peering_connection_id => peering_id
|
604
|
-
}
|
605
|
-
rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
606
|
-
route_table_ids: [rtb_id]
|
607
|
-
).route_tables.first
|
608
|
-
already_exists = false
|
609
|
-
rtbdesc.routes.each { |r|
|
610
|
-
if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
|
611
|
-
if r.vpc_peering_connection_id != peering_id
|
612
|
-
MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
|
613
|
-
raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
|
614
|
-
else
|
615
|
-
already_exists = true
|
616
|
-
end
|
617
|
-
end
|
618
|
-
}
|
619
|
-
next if already_exists
|
620
|
-
|
621
|
-
MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
|
622
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
|
623
|
-
} # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
|
624
|
-
|
625
|
-
begin
|
626
|
-
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
627
|
-
vpc_peering_connection_ids: [peering_id]
|
628
|
-
).vpc_peering_connections.first
|
629
|
-
|
630
|
-
if cnxn.status.code == "pending-acceptance"
|
631
|
-
if ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
|
632
|
-
MU.log "Auto-accepting peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
|
633
|
-
begin
|
634
|
-
MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).accept_vpc_peering_connection(
|
635
|
-
vpc_peering_connection_id: peering_id,
|
636
|
-
)
|
637
|
-
if peer['account'] != MU::Cloud::AWS.credToAcct(@config['credentials'])
|
638
|
-
# this seems to take a while across accounts
|
639
|
-
sleep 5
|
640
|
-
end
|
641
|
-
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
642
|
-
vpc_peering_connection_ids: [peering_id]
|
643
|
-
).vpc_peering_connections.first
|
644
|
-
rescue Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists => e
|
645
|
-
MU.log "Attempt to create duplicate peering connection to #{peer_id} from VPC #{@config['name']}", MU::WARN
|
646
|
-
end
|
647
|
-
|
648
|
-
# Create routes back from our new friend to us.
|
649
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer['vpc']['region'], credentials: peer['account']).uniq.each { |rtb_id|
|
650
|
-
peer_route_config = {
|
651
|
-
:route_table_id => rtb_id,
|
652
|
-
:destination_cidr_block => @config['ip_block'],
|
653
|
-
:vpc_peering_connection_id => peering_id
|
654
|
-
}
|
655
|
-
begin
|
656
|
-
resp = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).create_route(peer_route_config)
|
657
|
-
rescue Aws::EC2::Errors::RouteAlreadyExists => e
|
658
|
-
rtbdesc = MU::Cloud::AWS.ec2(region: peer['vpc']['region'], credentials: peer['account']).describe_route_tables(
|
659
|
-
route_table_ids: [rtb_id]
|
660
|
-
).route_tables.first
|
661
|
-
rtbdesc.routes.each { |r|
|
662
|
-
if r.destination_cidr_block == @config['ip_block']
|
663
|
-
if r.vpc_peering_connection_id != peering_id
|
664
|
-
MU.log "Attempt to create duplicate route to VPC #{@config['name']} (#{@config['ip_block']}) from peer VPC #{peer_id}'s route table #{rtb_id}", MU::ERR
|
665
|
-
end
|
666
|
-
end
|
667
|
-
}
|
668
|
-
end
|
669
|
-
}
|
670
|
-
else
|
671
|
-
MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
|
672
|
-
end
|
673
|
-
end
|
674
|
-
|
675
|
-
if cnxn.status.code == "failed" or cnxn.status.code == "rejected" or cnxn.status.code == "expired" or cnxn.status.code == "deleted"
|
676
|
-
MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer['vpc']['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
|
677
|
-
begin
|
678
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
|
679
|
-
vpc_peering_connection_id: peering_id
|
680
|
-
)
|
681
|
-
rescue Aws::EC2::Errors::InvalidStateTransition => e
|
682
|
-
# XXX apparently this is normal?
|
683
|
-
end
|
684
|
-
raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
|
685
|
-
end
|
686
|
-
|
687
|
-
end while cnxn.status.code != "active" and !((peer_obj.nil? or peer_obj.deploydata.nil? or !peer_obj.deploydata['auto_accept_peers']) and cnxn.status.code == "pending-acceptance")
|
688
|
-
|
239
|
+
peerWith(peer)
|
689
240
|
}
|
690
241
|
end
|
691
242
|
|
@@ -837,12 +388,9 @@ MU.log "wtf", MU::ERR, details: peer if peer_obj.nil? or peer_obj.first.nil?
|
|
837
388
|
if rtb_desc.associations
|
838
389
|
rtb_desc.associations.each { |assoc|
|
839
390
|
if assoc.subnet_id
|
840
|
-
if associations[assoc.subnet_id] and associations[assoc.subnet_id] != rtb['name']
|
841
|
-
MU.log "wait more than one route table association for #{assoc.subnet_id} what", MU::WARN, details: associations[assoc.subnet_id]+" => "+rtb['name']
|
842
|
-
end
|
843
391
|
associations[assoc.subnet_id] = rtb['name']
|
844
392
|
elsif assoc.gateway_id
|
845
|
-
MU.log "association I don't
|
393
|
+
MU.log " Saw a route table association I don't know how to adopt in #{@cloud_id}", MU::WARN, details: rtb_desc
|
846
394
|
end
|
847
395
|
}
|
848
396
|
end
|
@@ -902,20 +450,18 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
902
450
|
# Describe subnets associated with this VPC. We'll compose identifying
|
903
451
|
# information similar to what MU::Cloud.describe builds for first-class
|
904
452
|
# resources.
|
905
|
-
#
|
906
|
-
# that work like first-class objects. How would we enforce that?
|
907
|
-
# @return [Array<Hash>]: A list of cloud provider identifiers of subnets associated with this VPC.
|
453
|
+
# @return [Array<MU::Cloud::AWS::VPC::Subnet>]
|
908
454
|
def loadSubnets
|
909
|
-
if
|
910
|
-
|
911
|
-
|
912
|
-
|
913
|
-
]
|
914
|
-
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
455
|
+
return [] if !@cloud_id
|
456
|
+
|
457
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(
|
458
|
+
filters: [
|
459
|
+
{ name: "vpc-id", values: [@cloud_id] }
|
460
|
+
]
|
461
|
+
)
|
462
|
+
if resp.nil? or resp.subnets.nil? or resp.subnets.empty?
|
463
|
+
MU.log "Got empty results when trying to list subnets in #{@cloud_id}", MU::WARN
|
464
|
+
return []
|
919
465
|
end
|
920
466
|
|
921
467
|
@subnetcachesemaphore.synchronize {
|
@@ -926,23 +472,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
926
472
|
# metadata. Like ya do.
|
927
473
|
if !@config.nil? and @config.has_key?("subnets")
|
928
474
|
@config['subnets'].each { |subnet|
|
929
|
-
subnet['mu_name']
|
475
|
+
subnet['mu_name'] ||= @mu_name+"-"+subnet['name']
|
930
476
|
subnet['region'] = @config['region']
|
931
477
|
subnet['credentials'] = @config['credentials']
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
}
|
940
|
-
end
|
478
|
+
resp.subnets.each { |desc|
|
479
|
+
if desc.cidr_block == subnet["ip_block"]
|
480
|
+
subnet["tags"] = MU.structToHash(desc.tags)
|
481
|
+
subnet["cloud_id"] = desc.subnet_id
|
482
|
+
break
|
483
|
+
end
|
484
|
+
}
|
941
485
|
|
942
486
|
if subnet["cloud_id"] and !ext_ids.include?(subnet["cloud_id"])
|
943
487
|
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
944
488
|
elsif !subnet["cloud_id"]
|
945
|
-
resp.
|
489
|
+
resp.subnets.each { |desc|
|
946
490
|
if desc.cidr_block == subnet["ip_block"]
|
947
491
|
subnet['cloud_id'] = desc.subnet_id
|
948
492
|
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
@@ -955,21 +499,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
955
499
|
|
956
500
|
# Of course we might be loading up a dummy subnet object from a
|
957
501
|
# foreign or non-Mu-created VPC and subnet. So make something up.
|
958
|
-
if
|
959
|
-
resp.
|
960
|
-
subnet = {
|
961
|
-
|
962
|
-
|
502
|
+
if @subnets.empty?
|
503
|
+
resp.subnets.each { |desc|
|
504
|
+
subnet = {
|
505
|
+
"ip_block" => desc.cidr_block,
|
506
|
+
"tags" => MU.structToHash(desc.tags),
|
507
|
+
"cloud_id" => desc.subnet_id,
|
508
|
+
'region' => @config['region'],
|
509
|
+
'credentials' => @config['credentials'],
|
510
|
+
}
|
511
|
+
subnet['name'] = subnet["ip_block"].gsub(/[\.\/]/, "_")
|
963
512
|
subnet['mu_name'] = @mu_name+"-"+subnet['name']
|
964
|
-
|
965
|
-
subnet["cloud_id"] = desc.subnet_id
|
966
|
-
subnet['region'] = @config['region']
|
967
|
-
subnet['credentials'] = @config['credentials']
|
968
|
-
if !ext_ids.include?(desc.subnet_id)
|
969
|
-
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
970
|
-
end
|
513
|
+
@subnets << MU::Cloud::AWS::VPC::Subnet.new(self, subnet)
|
971
514
|
}
|
972
515
|
end
|
516
|
+
|
973
517
|
return @subnets
|
974
518
|
}
|
975
519
|
end
|
@@ -1138,14 +682,14 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1138
682
|
MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, region: MU.myRegion)
|
1139
683
|
MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, region: region, credentials: credentials)
|
1140
684
|
|
1141
|
-
if MU::Cloud::AWS::VPC.
|
685
|
+
if MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
|
1142
686
|
return true
|
1143
687
|
else
|
1144
688
|
# The cache can be out of date at times, check again without it
|
1145
689
|
MU::Cloud::AWS::VPC.update_route_tables_cache(my_subnets_key, use_cache: false, region: MU.myRegion)
|
1146
690
|
MU::Cloud::AWS::VPC.update_route_tables_cache(target_subnets_key, use_cache: false, region: region, credentials: credentials)
|
1147
691
|
|
1148
|
-
return MU::Cloud::AWS::VPC.
|
692
|
+
return MU::Cloud::AWS::VPC.can_route_to_master_peer?(my_subnets_key, target_subnets_key, instance_id)
|
1149
693
|
end
|
1150
694
|
|
1151
695
|
end
|
@@ -1184,7 +728,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1184
728
|
# @param target_subnets_key [String]: The subnet/subnets on the other side of the peered VPC.
|
1185
729
|
# @param instance_id [String]: The instance ID in the target subnet/subnets.
|
1186
730
|
# @return [Boolean]
|
1187
|
-
def self.
|
731
|
+
def self.can_route_to_master_peer?(source_subnets_key, target_subnets_key, instance_id)
|
1188
732
|
my_routes = []
|
1189
733
|
vpc_peer_mapping = {}
|
1190
734
|
|
@@ -1278,32 +822,39 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1278
822
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
1279
823
|
# @param region [String]: The cloud provider region
|
1280
824
|
# @return [void]
|
1281
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
825
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
1282
826
|
MU.log "AWS::VPC.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
1283
827
|
|
1284
828
|
tagfilters = [
|
1285
|
-
{name: "tag:MU-ID", values: [
|
829
|
+
{name: "tag:MU-ID", values: [deploy_id]}
|
1286
830
|
]
|
1287
831
|
if !ignoremaster
|
1288
832
|
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
1289
833
|
end
|
1290
834
|
|
1291
835
|
vpcs = []
|
1292
|
-
|
1293
|
-
begin
|
836
|
+
MU.retrier([Aws::EC2::Errors::InvalidVpcIDNotFound], wait: 5) {
|
1294
837
|
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs(filters: tagfilters, max_results: 1000).vpcs
|
1295
838
|
vpcs = resp if !resp.empty?
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1301
|
-
|
1302
|
-
|
839
|
+
}
|
840
|
+
|
841
|
+
# resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
842
|
+
# filters: [
|
843
|
+
# {
|
844
|
+
# name: "requester-vpc-info.vpc-id",
|
845
|
+
# values: [@cloud_id]
|
846
|
+
# },
|
847
|
+
# {
|
848
|
+
# name: "accepter-vpc-info.vpc-id",
|
849
|
+
# values: [peer_id.to_s]
|
850
|
+
# }
|
851
|
+
# ]
|
852
|
+
# )
|
1303
853
|
|
1304
854
|
if !vpcs.empty?
|
1305
855
|
gwthreads = []
|
1306
856
|
vpcs.each { |vpc|
|
857
|
+
purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
|
1307
858
|
# NAT gateways don't have any tags, and we can't assign them a name. Lets find them based on a VPC ID
|
1308
859
|
gwthreads << Thread.new {
|
1309
860
|
purge_nat_gateways(noop, vpc_id: vpc.vpc_id, region: region, credentials: credentials)
|
@@ -1325,7 +876,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1325
876
|
|
1326
877
|
# unless noop
|
1327
878
|
# MU::Cloud::AWS.iam.list_roles.roles.each{ |role|
|
1328
|
-
# match_string = "#{
|
879
|
+
# match_string = "#{deploy_id}.*TRAFFIC-LOG"
|
1329
880
|
# }
|
1330
881
|
# end
|
1331
882
|
end
|
@@ -1379,11 +930,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1379
930
|
logdesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
|
1380
931
|
# logdesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
|
1381
932
|
configurator.insertKitten(logdesc, "logs")
|
1382
|
-
vpc['
|
1383
|
-
vpc['dependencies'] << {
|
1384
|
-
"type" => "log",
|
1385
|
-
"name" => vpc['name']+"loggroup"
|
1386
|
-
}
|
933
|
+
MU::Config.addDependency(vpc, vpc['name']+"loggroup", "log")
|
1387
934
|
|
1388
935
|
roledesc = {
|
1389
936
|
"name" => vpc['name']+"logrole",
|
@@ -1421,11 +968,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1421
968
|
roledesc["tags"] = vpc["tags"] if !vpc["tags"].nil?
|
1422
969
|
roledesc["optional_tags"] = vpc["optional_tags"] if !vpc["optional_tags"].nil?
|
1423
970
|
configurator.insertKitten(roledesc, "roles")
|
1424
|
-
vpc['
|
1425
|
-
vpc['dependencies'] << {
|
1426
|
-
"type" => "role",
|
1427
|
-
"name" => vpc['name']+"logrole"
|
1428
|
-
}
|
971
|
+
MU::Config.addDependency(vpc, vpc['name']+"logrole", "role")
|
1429
972
|
end
|
1430
973
|
|
1431
974
|
subnet_routes = Hash.new
|
@@ -1476,10 +1019,7 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1476
1019
|
subnet_routes[table['name']].each { |subnet|
|
1477
1020
|
nat_routes[subnet] = route['nat_host_name']
|
1478
1021
|
}
|
1479
|
-
vpc['
|
1480
|
-
"type" => "server",
|
1481
|
-
"name" => route['nat_host_name']
|
1482
|
-
}
|
1022
|
+
MU::Config.addDependency(vpc, route['nat_host_name'], "server", no_create_wait: true)
|
1483
1023
|
elsif route['gateway'] == '#NAT'
|
1484
1024
|
vpc['create_nat_gateway'] = true
|
1485
1025
|
private_rtbs << table['name']
|
@@ -1492,17 +1032,11 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1492
1032
|
if route['gateway'] == '#INTERNET'
|
1493
1033
|
if table['name'] == subnet['route_table']
|
1494
1034
|
subnet['is_public'] = true
|
1495
|
-
if vpc['create_nat_gateway']
|
1496
|
-
|
1497
|
-
|
1498
|
-
|
1499
|
-
|
1500
|
-
subnet['create_nat_gateway'] = false
|
1501
|
-
else
|
1502
|
-
subnet['create_nat_gateway'] = true
|
1503
|
-
nat_gateway_added = true
|
1504
|
-
end
|
1505
|
-
end
|
1035
|
+
if vpc['create_nat_gateway'] and (vpc['nat_gateway_multi_az'] or !nat_gateway_added)
|
1036
|
+
subnet['create_nat_gateway'] = true
|
1037
|
+
nat_gateway_added = true
|
1038
|
+
else
|
1039
|
+
subnet['create_nat_gateway'] = false
|
1506
1040
|
end
|
1507
1041
|
else
|
1508
1042
|
subnet['is_public'] = false
|
@@ -1662,17 +1196,10 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1662
1196
|
|
1663
1197
|
ifaces.each { |iface|
|
1664
1198
|
if iface.vpc_id
|
1665
|
-
|
1666
|
-
|
1667
|
-
|
1668
|
-
|
1669
|
-
]
|
1670
|
-
).security_groups
|
1671
|
-
if default_sg_resp and default_sg_resp.size == 1
|
1672
|
-
default_sg = default_sg_resp.first.group_id
|
1673
|
-
if iface.groups.size > 1 or
|
1674
|
-
iface.groups.first.group_id != default_sg
|
1675
|
-
MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
|
1199
|
+
default_sg = MU::Cloud::AWS::VPC.getDefaultSg(iface.vpc_id, region: region, credentials: credentials)
|
1200
|
+
if default_sg and (iface.groups.size > 1 or (iface.groups.size == 1 and iface.groups.first.group_id != default_sg))
|
1201
|
+
MU.log "Removing extra security groups from ENI #{iface.network_interface_id}"
|
1202
|
+
if !noop
|
1676
1203
|
begin
|
1677
1204
|
MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
|
1678
1205
|
network_interface_id: iface.network_interface_id,
|
@@ -1693,11 +1220,15 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1693
1220
|
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
1694
1221
|
MU.log "Can't detach #{iface.network_interface_id}: #{e.message}", MU::WARN, details: iface.attachment
|
1695
1222
|
next
|
1223
|
+
rescue Aws::EC2::Errors::IncorrectState => e
|
1224
|
+
MU.log e.message, MU::WARN
|
1225
|
+
sleep 5
|
1226
|
+
retry
|
1696
1227
|
rescue Aws::EC2::Errors::InvalidAttachmentIDNotFound => e
|
1697
1228
|
# suits me just fine
|
1698
1229
|
rescue Aws::EC2::Errors::AuthFailure => e
|
1699
1230
|
if !tried_lbs and iface.attachment.instance_owner_id == "amazon-elb"
|
1700
|
-
MU::Cloud
|
1231
|
+
MU::Cloud.resourceClass("AWS", "LoadBalancer").cleanup(
|
1701
1232
|
noop: noop,
|
1702
1233
|
region: region,
|
1703
1234
|
credentials: credentials,
|
@@ -1719,9 +1250,230 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1719
1250
|
}
|
1720
1251
|
end
|
1721
1252
|
|
1253
|
+
# Fetch the group id of the +default+ security group for the given VPC
|
1254
|
+
# @param vpc_id [String]
|
1255
|
+
# @param region [String]
|
1256
|
+
# @param credentials [String]
|
1257
|
+
# @return [String]
|
1258
|
+
def self.getDefaultSg(vpc_id, region: MU.curRegion, credentials: nil)
|
1259
|
+
default_sg_resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
|
1260
|
+
filters: [
|
1261
|
+
{ name: "group-name", values: ["default"] },
|
1262
|
+
{ name: "vpc-id", values: [vpc_id] }
|
1263
|
+
]
|
1264
|
+
).security_groups
|
1265
|
+
if default_sg_resp and default_sg_resp.size == 1
|
1266
|
+
return default_sg_resp.first.group_id
|
1267
|
+
end
|
1268
|
+
nil
|
1269
|
+
end
|
1270
|
+
|
1271
|
+
# Try to locate the default VPC for a region, and return a BoK-style
|
1272
|
+
# config fragment for something that might want to live in it.
|
1273
|
+
def self.defaultVpc(region, credentials)
|
1274
|
+
cfg_fragment = nil
|
1275
|
+
MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_vpcs.vpcs.each { |vpc|
|
1276
|
+
if vpc.is_default
|
1277
|
+
cfg_fragment = {
|
1278
|
+
"id" => vpc.vpc_id,
|
1279
|
+
"cloud" => "AWS",
|
1280
|
+
"region" => region,
|
1281
|
+
"credentials" => credentials
|
1282
|
+
}
|
1283
|
+
cfg_fragment['subnets'] = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_subnets(
|
1284
|
+
filters: [
|
1285
|
+
{
|
1286
|
+
name: "vpc-id",
|
1287
|
+
values: [vpc.vpc_id]
|
1288
|
+
}
|
1289
|
+
]
|
1290
|
+
).subnets.map { |s| { "subnet_id" => s.subnet_id } }
|
1291
|
+
break
|
1292
|
+
end
|
1293
|
+
}
|
1294
|
+
|
1295
|
+
cfg_fragment
|
1296
|
+
end
|
1297
|
+
|
1298
|
+
# Return a {MU::Config::Ref} that indicates this VPC.
|
1299
|
+
# @param subnet_ids [Array<String>]: Optional list of subnet ids with which to infer a +subnet_pref+ parameter.
|
1300
|
+
# @return [MU::Config::Ref]
|
1301
|
+
def getReference(subnet_ids = [])
|
1302
|
+
have_private = have_public = false
|
1303
|
+
subnets.each { |s|
|
1304
|
+
next if subnet_ids and !subnet_ids.empty? and !subnet_ids.include?(s.cloud_id)
|
1305
|
+
if s.private?
|
1306
|
+
have_private = true
|
1307
|
+
else
|
1308
|
+
have_public = true
|
1309
|
+
end
|
1310
|
+
}
|
1311
|
+
subnet_pref = if have_private == have_public
|
1312
|
+
"any"
|
1313
|
+
elsif have_private
|
1314
|
+
"all_private"
|
1315
|
+
elsif have_public
|
1316
|
+
"all_public"
|
1317
|
+
end
|
1318
|
+
MU::Config::Ref.get(
|
1319
|
+
id: @cloud_id,
|
1320
|
+
cloud: "AWS",
|
1321
|
+
credentials: @credentials,
|
1322
|
+
region: @config['region'],
|
1323
|
+
type: "vpcs",
|
1324
|
+
subnet_pref: subnet_pref
|
1325
|
+
)
|
1326
|
+
end
|
1722
1327
|
|
1723
1328
|
private
|
1724
1329
|
|
1330
|
+
def peerWith(peer)
|
1331
|
+
peer_ref = MU::Config::Ref.get(peer['vpc'])
|
1332
|
+
peer_obj = peer_ref.kitten
|
1333
|
+
peer_id = peer_ref.kitten.cloud_id
|
1334
|
+
if peer_id == @cloud_id
|
1335
|
+
MU.log "#{@mu_name} attempted to peer with itself (#{@cloud_id})", MU::ERR, details: peer
|
1336
|
+
raise "#{@mu_name} attempted to peer with itself (#{@cloud_id})"
|
1337
|
+
end
|
1338
|
+
|
1339
|
+
if peer_obj and peer_obj.config['peers']
|
1340
|
+
peer_obj.config['peers'].each { |peerpeer|
|
1341
|
+
if peerpeer['vpc']['name'] == @config['name'] and
|
1342
|
+
(peer['vpc']['name'] <=> @config['name']) == -1
|
1343
|
+
MU.log "VPCs #{peer['vpc']['name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
|
1344
|
+
return
|
1345
|
+
# XXX and if deploy_id matches or is unset
|
1346
|
+
end
|
1347
|
+
}
|
1348
|
+
|
1349
|
+
peer['account'] ||= MU::Cloud::AWS.credToAcct(peer_obj.credentials)
|
1350
|
+
end
|
1351
|
+
|
1352
|
+
peer['account'] ||= MU::Cloud::AWS.account_number
|
1353
|
+
|
1354
|
+
# See if the peering connection exists before we bother
|
1355
|
+
# creating it.
|
1356
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
1357
|
+
filters: [
|
1358
|
+
{
|
1359
|
+
name: "requester-vpc-info.vpc-id",
|
1360
|
+
values: [@cloud_id]
|
1361
|
+
},
|
1362
|
+
{
|
1363
|
+
name: "accepter-vpc-info.vpc-id",
|
1364
|
+
values: [peer_id.to_s]
|
1365
|
+
}
|
1366
|
+
]
|
1367
|
+
)
|
1368
|
+
|
1369
|
+
peering_id = if !resp or !resp.vpc_peering_connections or
|
1370
|
+
resp.vpc_peering_connections.empty?
|
1371
|
+
|
1372
|
+
MU.log "Setting peering connection from VPC #{@config['name']} (#{@cloud_id} in account #{MU::Cloud::AWS.credToAcct(@config['credentials'])}) to #{peer_id} in account #{peer['account']}", details: peer
|
1373
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_vpc_peering_connection(
|
1374
|
+
vpc_id: @cloud_id,
|
1375
|
+
peer_vpc_id: peer_id,
|
1376
|
+
peer_owner_id: peer['account'],
|
1377
|
+
peer_region: peer_obj.config['region']
|
1378
|
+
)
|
1379
|
+
resp.vpc_peering_connection.vpc_peering_connection_id
|
1380
|
+
else
|
1381
|
+
resp.vpc_peering_connections.first.vpc_peering_connection_id
|
1382
|
+
end
|
1383
|
+
|
1384
|
+
peering_name = @deploy.getResourceName(@config['name']+"-PEER-"+peer_id)
|
1385
|
+
|
1386
|
+
tag_me(peering_id, peering_name)
|
1387
|
+
|
1388
|
+
# Create routes to our new friend.
|
1389
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(@cloud_id, region: @config['region'], credentials: @config['credentials']).each { |rtb_id|
|
1390
|
+
my_route_config = {
|
1391
|
+
:route_table_id => rtb_id,
|
1392
|
+
:destination_cidr_block => peer_obj.cloud_desc.cidr_block,
|
1393
|
+
:vpc_peering_connection_id => peering_id
|
1394
|
+
}
|
1395
|
+
rtbdesc = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
1396
|
+
route_table_ids: [rtb_id]
|
1397
|
+
).route_tables.first
|
1398
|
+
already_exists = false
|
1399
|
+
rtbdesc.routes.each { |r|
|
1400
|
+
if r.destination_cidr_block == peer_obj.cloud_desc.cidr_block
|
1401
|
+
if r.vpc_peering_connection_id != peering_id
|
1402
|
+
MU.log "Attempt to create duplicate route to #{peer_obj.cloud_desc.cidr_block} from VPC #{@config['name']}", MU::ERR, details: r
|
1403
|
+
raise MuError, "Can't create route via #{peering_id}, a route to #{peer_obj.cloud_desc.cidr_block} already exists"
|
1404
|
+
else
|
1405
|
+
already_exists = true
|
1406
|
+
end
|
1407
|
+
end
|
1408
|
+
}
|
1409
|
+
next if already_exists
|
1410
|
+
|
1411
|
+
MU.log "Creating peering route to #{peer_obj.cloud_desc.cidr_block} in #{peer['vpc']['region']} from VPC #{@config['name']} in #{@config['region']}"
|
1412
|
+
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route(my_route_config)
|
1413
|
+
} # MU::Cloud::AWS::VPC.listAllSubnetRouteTables
|
1414
|
+
|
1415
|
+
can_auto_accept = ((!peer_obj.nil? and !peer_obj.deploydata.nil? and peer_obj.deploydata['auto_accept_peers']) or $MU_CFG['allow_invade_foreign_vpcs'])
|
1416
|
+
|
1417
|
+
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
1418
|
+
vpc_peering_connection_ids: [peering_id]
|
1419
|
+
).vpc_peering_connections.first
|
1420
|
+
|
1421
|
+
loop_if = Proc.new {
|
1422
|
+
cnxn = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_vpc_peering_connections(
|
1423
|
+
vpc_peering_connection_ids: [peering_id]
|
1424
|
+
).vpc_peering_connections.first
|
1425
|
+
((can_auto_accept and cnxn.status.code == "pending-acceptance") or (cnxn.status.code != "active" and cnxn.status.code != "pending-acceptance"))
|
1426
|
+
}
|
1427
|
+
|
1428
|
+
MU.retrier(wait: 5, loop_if: loop_if, ignoreme: [Aws::EC2::Errors::VpcPeeringConnectionAlreadyExists, Aws::EC2::Errors::RouteAlreadyExists]) {
|
1429
|
+
if cnxn.status.code == "pending-acceptance"
|
1430
|
+
if can_auto_accept
|
1431
|
+
MU.log "Auto-accepting peering connection #{peering_id} from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id}", MU::NOTICE
|
1432
|
+
MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).accept_vpc_peering_connection(
|
1433
|
+
vpc_peering_connection_id: peering_id,
|
1434
|
+
)
|
1435
|
+
|
1436
|
+
# Create routes back from our new friend to us.
|
1437
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_id, region: peer_obj.config['region'], credentials: peer['account']).uniq.each { |rtb_id|
|
1438
|
+
peer_route_config = {
|
1439
|
+
:route_table_id => rtb_id,
|
1440
|
+
:destination_cidr_block => @config['ip_block'],
|
1441
|
+
:vpc_peering_connection_id => peering_id
|
1442
|
+
}
|
1443
|
+
resp = MU::Cloud::AWS.ec2(region: peer_obj.config['region'], credentials: peer['account']).create_route(peer_route_config)
|
1444
|
+
}
|
1445
|
+
else
|
1446
|
+
MU.log "VPC #{peer_id} is not managed by this Mu server or is not configured to auto-accept peering requests. You must accept the peering request for '#{@config['name']}' (#{@cloud_id}) by hand.", MU::WARN, details: "In the AWS Console, go to VPC => Peering Connections and look in the Actions drop-down. You can also set 'Invade Foreign VPCs' to 'true' using mu-configure to auto-accept all peering connections within this account, regardless of whether this Mu server owns the VPCs. This setting is per-user."
|
1447
|
+
end
|
1448
|
+
end
|
1449
|
+
|
1450
|
+
if ["failed", "rejected", "expired", "deleted"].include?(cnxn.status.code)
|
1451
|
+
MU.log "VPC peering connection from VPC #{@config['name']} (#{@cloud_id} in #{@config['region']}) to #{peer_id} in #{peer_obj.config['region']} #{cnxn.status.code}: #{cnxn.status.message}", MU::ERR
|
1452
|
+
begin
|
1453
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).delete_vpc_peering_connection(
|
1454
|
+
vpc_peering_connection_id: peering_id
|
1455
|
+
)
|
1456
|
+
rescue Aws::EC2::Errors::InvalidStateTransition
|
1457
|
+
# XXX apparently this is normal?
|
1458
|
+
end
|
1459
|
+
raise MuError, "VPC peering connection from VPC #{@config['name']} (#{@cloud_id}) to #{peer_id} #{cnxn.status.code}: #{cnxn.status.message}"
|
1460
|
+
end
|
1461
|
+
|
1462
|
+
}
|
1463
|
+
|
1464
|
+
end
|
1465
|
+
|
1466
|
+
def tag_me(resource_id = @cloud_id, name = @mu_name)
|
1467
|
+
MU::Cloud::AWS.createStandardTags(
|
1468
|
+
resource_id,
|
1469
|
+
region: @config['region'],
|
1470
|
+
credentials: @config['credentials'],
|
1471
|
+
optional: @config['optional_tags'],
|
1472
|
+
nametag: name,
|
1473
|
+
othertags: @config['tags']
|
1474
|
+
)
|
1475
|
+
end
|
1476
|
+
|
1725
1477
|
# Helper method for manufacturing route tables. Expect to be called from
|
1726
1478
|
# {MU::Cloud::AWS::VPC#create} or {MU::Cloud::AWS::VPC#groom}.
|
1727
1479
|
# @param rtb [Hash]: A route table description parsed through {MU::Config::BasketofKittens::vpcs::route_tables}.
|
@@ -1733,21 +1485,9 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1733
1485
|
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_route_table(vpc_id: vpc_id).route_table
|
1734
1486
|
route_table_id = rtb['route_table_id'] = resp.route_table_id
|
1735
1487
|
sleep 5
|
1736
|
-
MU::Cloud::AWS.createTag(route_table_id, "Name", vpc_name+"-"+rtb['name'].upcase, credentials: @config['credentials'])
|
1737
|
-
|
1738
|
-
if @config['tags']
|
1739
|
-
@config['tags'].each { |tag|
|
1740
|
-
MU::Cloud::AWS.createTag(route_table_id, tag['key'], tag['value'], credentials: @config['credentials'])
|
1741
|
-
}
|
1742
|
-
end
|
1743
1488
|
|
1744
|
-
|
1745
|
-
MU::MommaCat.listOptionalTags.each { |key, value|
|
1746
|
-
MU::Cloud::AWS.createTag(route_table_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
1747
|
-
}
|
1748
|
-
end
|
1489
|
+
tag_me(route_table_id, vpc_name+"-"+rtb['name'].upcase)
|
1749
1490
|
|
1750
|
-
MU::Cloud::AWS.createStandardTags(route_table_id, credentials: @config['credentials'])
|
1751
1491
|
rtb['routes'].each { |route|
|
1752
1492
|
if route['nat_host_id'].nil? and route['nat_host_name'].nil?
|
1753
1493
|
route_config = {
|
@@ -1840,36 +1580,25 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1840
1580
|
).nat_gateways
|
1841
1581
|
|
1842
1582
|
threads = []
|
1843
|
-
|
1583
|
+
|
1844
1584
|
if !gateways.empty?
|
1845
1585
|
gateways.each { |gateway|
|
1586
|
+
next if noop
|
1587
|
+
MU.log "Deleting NAT Gateway #{gateway.nat_gateway_id}"
|
1846
1588
|
threads << Thread.new {
|
1847
|
-
MU.
|
1848
|
-
|
1849
|
-
|
1850
|
-
|
1851
|
-
|
1852
|
-
|
1589
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_nat_gateway(nat_gateway_id: gateway.nat_gateway_id)
|
1590
|
+
|
1591
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
1592
|
+
|
1593
|
+
loop_if = Proc.new {
|
1594
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
1595
|
+
(resp.state != "deleted" and resp.state != "failed")
|
1596
|
+
}
|
1597
|
+
|
1598
|
+
MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::NatGatewayMalformed, Aws::EC2::Errors::NatGatewayNotFound], max: 50, loop_if: loop_if) { |retries, _wait|
|
1599
|
+
MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if retries % 3 == 0
|
1600
|
+
}
|
1853
1601
|
|
1854
|
-
attempts = 0
|
1855
|
-
while resp.state != "deleted" and resp.state != "failed"
|
1856
|
-
MU.log "Waiting for nat gateway #{gateway.nat_gateway_id} to delete" if attempts % 2 == 0
|
1857
|
-
sleep 30
|
1858
|
-
begin
|
1859
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_nat_gateways(nat_gateway_ids: [gateway.nat_gateway_id]).nat_gateways.first
|
1860
|
-
rescue Aws::EmptyStructure, NoMethodError
|
1861
|
-
sleep 5
|
1862
|
-
retry
|
1863
|
-
rescue Aws::EC2::Errors::NatGatewayNotFound
|
1864
|
-
MU.log "NAT gateway #{gateway.nat_gateway_id} already deleted", MU::NOTICE
|
1865
|
-
end
|
1866
|
-
MU.log "Timed out while waiting for NAT Gateway to delete #{gateway.nat_gateway_id}: #{resp}", MU::WARN if attempts > 50
|
1867
|
-
attempts += 1
|
1868
|
-
end
|
1869
|
-
rescue Aws::EC2::Errors::NatGatewayMalformed
|
1870
|
-
MU.log "NAT Gateway #{gateway.nat_gateway_id} was already deleted", MU::NOTICE
|
1871
|
-
end
|
1872
|
-
end
|
1873
1602
|
}
|
1874
1603
|
}
|
1875
1604
|
end
|
@@ -1898,38 +1627,21 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
1898
1627
|
).vpc_endpoints
|
1899
1628
|
|
1900
1629
|
threads = []
|
1901
|
-
|
1630
|
+
|
1902
1631
|
if !vpc_endpoints.empty?
|
1903
1632
|
vpc_endpoints.each { |endpoint|
|
1633
|
+
MU.log "Deleting VPC endpoint #{endpoint.vpc_endpoint_id}"
|
1634
|
+
next if noop
|
1904
1635
|
threads << Thread.new {
|
1905
|
-
MU.
|
1906
|
-
MU.
|
1907
|
-
|
1908
|
-
|
1909
|
-
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
1913
|
-
|
1914
|
-
MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if attempts % 5 == 0
|
1915
|
-
sleep 30
|
1916
|
-
begin
|
1917
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
1918
|
-
rescue Aws::EmptyStructure, NoMethodError
|
1919
|
-
sleep 5
|
1920
|
-
retry
|
1921
|
-
rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
|
1922
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
|
1923
|
-
end
|
1924
|
-
MU.log "Timed out while waiting for VPC endpoint to delete #{endpoint.vpc_endpoint_id}: #{resp}", MU::WARN if attempts > 50
|
1925
|
-
attempts += 1
|
1926
|
-
end
|
1927
|
-
rescue Aws::EC2::Errors::VpcEndpointIdMalformed
|
1928
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} was already deleted", MU::NOTICE
|
1929
|
-
rescue Aws::EC2::Errors::InvalidVpcEndpointIdNotFound
|
1930
|
-
MU.log "VPC endpoint #{endpoint.vpc_endpoint_id} already deleted", MU::NOTICE
|
1931
|
-
end
|
1932
|
-
end
|
1636
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id])
|
1637
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
1638
|
+
loop_if = Proc.new {
|
1639
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_endpoints(vpc_endpoint_ids: [endpoint.vpc_endpoint_id]).vpc_endpoints.first
|
1640
|
+
resp.state != "deleted"
|
1641
|
+
}
|
1642
|
+
MU.retrier([Aws::EmptyStructure, NoMethodError], ignoreme: [Aws::EC2::Errors::InvalidVpcEndpointIdNotFound, Aws::EC2::Errors::VpcEndpointIdMalformed], max: 20, wait: 10, loop_if: loop_if) { |retries, _wait|
|
1643
|
+
MU.log "Waiting for VPC endpoint #{endpoint.vpc_endpoint_id} to delete" if retries % 5 == 0
|
1644
|
+
}
|
1933
1645
|
}
|
1934
1646
|
}
|
1935
1647
|
end
|
@@ -2001,56 +1713,6 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
2001
1713
|
end
|
2002
1714
|
private_class_method :purge_routetables
|
2003
1715
|
|
2004
|
-
# Remove all subnets associated with the currently loaded deployment.
|
2005
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
2006
|
-
# @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
|
2007
|
-
# @param region [String]: The cloud provider region
|
2008
|
-
# @return [void]
|
2009
|
-
def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
|
2010
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_subnets(
|
2011
|
-
filters: tagfilters
|
2012
|
-
)
|
2013
|
-
subnets = resp.data.subnets
|
2014
|
-
|
2015
|
-
return if subnets.nil? or subnets.size == 0
|
2016
|
-
|
2017
|
-
retries = 0
|
2018
|
-
subnets.each { |subnet|
|
2019
|
-
MU.log "Deleting Subnet #{subnet.subnet_id}"
|
2020
|
-
begin
|
2021
|
-
if subnet.state != "available"
|
2022
|
-
MU.log "Waiting for #{subnet.subnet_id} to be in a removable state...", MU::NOTICE
|
2023
|
-
sleep 30
|
2024
|
-
else
|
2025
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_subnet(subnet_id: subnet.subnet_id) if !noop
|
2026
|
-
end
|
2027
|
-
rescue Aws::EC2::Errors::DependencyViolation => e
|
2028
|
-
# We're often stuck waiting for an RDS database or something else
|
2029
|
-
# that takes 5-ever to delete.
|
2030
|
-
if retries < 19
|
2031
|
-
loglevel = (retries > 0 and (retries % 3) == 0) ? MU::NOTICE : MU::DEBUG
|
2032
|
-
MU.log "#{e.message} (retry #{retries.to_s}/20)", loglevel
|
2033
|
-
if loglevel == MU::NOTICE
|
2034
|
-
MU::Cloud::AWS::VPC.purge_interfaces(noop, [{name: "subnet-id", values: [subnet.subnet_id]}], region: region, credentials: credentials)
|
2035
|
-
end
|
2036
|
-
sleep 30
|
2037
|
-
retries = retries + 1
|
2038
|
-
retry
|
2039
|
-
elsif retries < 20
|
2040
|
-
MU.log "#{e.message} (final attempt)", MU::WARN
|
2041
|
-
sleep 60
|
2042
|
-
retries = retries + 1
|
2043
|
-
retry
|
2044
|
-
else
|
2045
|
-
raise e
|
2046
|
-
end
|
2047
|
-
rescue Aws::EC2::Errors::InvalidSubnetIDNotFound
|
2048
|
-
next
|
2049
|
-
end while subnet.state != "available"
|
2050
|
-
}
|
2051
|
-
end
|
2052
|
-
private_class_method :purge_subnets
|
2053
|
-
|
2054
1716
|
# Remove all DHCP options sets associated with the currently loaded
|
2055
1717
|
# deployment.
|
2056
1718
|
# @param noop [Boolean]: If true, will only print what would be done
|
@@ -2081,6 +1743,61 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
2081
1743
|
end
|
2082
1744
|
private_class_method :purge_dhcpopts
|
2083
1745
|
|
1746
|
+
def self.purge_peering_connections(noop, vpc_id, region: MU.curRegion, credentials: nil)
|
1747
|
+
my_peer_conns = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
|
1748
|
+
filters: [
|
1749
|
+
{
|
1750
|
+
name: "requester-vpc-info.vpc-id",
|
1751
|
+
values: [vpc_id]
|
1752
|
+
}
|
1753
|
+
]
|
1754
|
+
).vpc_peering_connections
|
1755
|
+
my_peer_conns.concat(MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpc_peering_connections(
|
1756
|
+
filters: [
|
1757
|
+
{
|
1758
|
+
name: "accepter-vpc-info.vpc-id",
|
1759
|
+
values: [vpc_id]
|
1760
|
+
}
|
1761
|
+
]
|
1762
|
+
).vpc_peering_connections)
|
1763
|
+
|
1764
|
+
my_peer_conns.each { |cnxn|
|
1765
|
+
[cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
|
1766
|
+
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
|
1767
|
+
begin
|
1768
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
|
1769
|
+
route_table_ids: [rtb_id]
|
1770
|
+
)
|
1771
|
+
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
|
1772
|
+
next
|
1773
|
+
end
|
1774
|
+
resp.route_tables.each { |rtb|
|
1775
|
+
rtb.routes.each { |route|
|
1776
|
+
if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
|
1777
|
+
MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
|
1778
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
|
1779
|
+
route_table_id: rtb_id,
|
1780
|
+
destination_cidr_block: route.destination_cidr_block
|
1781
|
+
) if !noop
|
1782
|
+
end
|
1783
|
+
}
|
1784
|
+
}
|
1785
|
+
}
|
1786
|
+
}
|
1787
|
+
MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
|
1788
|
+
begin
|
1789
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
|
1790
|
+
vpc_peering_connection_id: cnxn.vpc_peering_connection_id
|
1791
|
+
) if !noop
|
1792
|
+
rescue Aws::EC2::Errors::InvalidStateTransition
|
1793
|
+
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
|
1794
|
+
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
1795
|
+
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
|
1796
|
+
end
|
1797
|
+
}
|
1798
|
+
end
|
1799
|
+
private_class_method :purge_peering_connections
|
1800
|
+
|
2084
1801
|
# Remove all VPCs associated with the currently loaded deployment.
|
2085
1802
|
# @param noop [Boolean]: If true, will only print what would be done
|
2086
1803
|
# @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
|
@@ -2088,177 +1805,40 @@ MU.log "association I don't understand in #{@cloud_id}", MU::WARN, details: rtb_
|
|
2088
1805
|
# @return [void]
|
2089
1806
|
def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion, credentials: nil)
|
2090
1807
|
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_vpcs(
|
2091
|
-
|
1808
|
+
filters: tagfilters
|
2092
1809
|
)
|
2093
1810
|
|
2094
1811
|
vpcs = resp.data.vpcs
|
2095
1812
|
return if vpcs.nil? or vpcs.size == 0
|
2096
1813
|
|
2097
1814
|
vpcs.each { |vpc|
|
2098
|
-
|
2099
|
-
|
2100
|
-
|
2101
|
-
|
2102
|
-
|
2103
|
-
|
2104
|
-
|
2105
|
-
|
2106
|
-
|
2107
|
-
|
2108
|
-
{
|
2109
|
-
name: "accepter-vpc-info.vpc-id",
|
2110
|
-
values: [vpc.vpc_id]
|
2111
|
-
}
|
2112
|
-
]
|
2113
|
-
).vpc_peering_connections)
|
2114
|
-
my_peer_conns.each { |cnxn|
|
2115
|
-
|
2116
|
-
[cnxn.accepter_vpc_info.vpc_id, cnxn.requester_vpc_info.vpc_id].each { |peer_vpc|
|
2117
|
-
MU::Cloud::AWS::VPC.listAllSubnetRouteTables(peer_vpc, region: region, credentials: credentials).each { |rtb_id|
|
2118
|
-
begin
|
2119
|
-
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_route_tables(
|
2120
|
-
route_table_ids: [rtb_id]
|
2121
|
-
)
|
2122
|
-
rescue Aws::EC2::Errors::InvalidRouteTableIDNotFound
|
2123
|
-
next
|
2124
|
-
end
|
2125
|
-
resp.route_tables.each { |rtb|
|
2126
|
-
rtb.routes.each { |route|
|
2127
|
-
if route.vpc_peering_connection_id == cnxn.vpc_peering_connection_id
|
2128
|
-
MU.log "Removing route #{route.destination_cidr_block} from route table #{rtb_id} in VPC #{peer_vpc}"
|
2129
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_route(
|
2130
|
-
route_table_id: rtb_id,
|
2131
|
-
destination_cidr_block: route.destination_cidr_block
|
2132
|
-
) if !noop
|
2133
|
-
end
|
2134
|
-
}
|
2135
|
-
}
|
2136
|
-
}
|
2137
|
-
}
|
2138
|
-
MU.log "Deleting VPC peering connection #{cnxn.vpc_peering_connection_id}"
|
2139
|
-
begin
|
2140
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc_peering_connection(
|
2141
|
-
vpc_peering_connection_id: cnxn.vpc_peering_connection_id
|
2142
|
-
) if !noop
|
2143
|
-
rescue Aws::EC2::Errors::InvalidStateTransition
|
2144
|
-
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} not in removable (state #{cnxn.status.code})", MU::WARN
|
2145
|
-
rescue Aws::EC2::Errors::OperationNotPermitted => e
|
2146
|
-
MU.log "VPC peering connection #{cnxn.vpc_peering_connection_id} refuses to delete: #{e.message}", MU::WARN
|
2147
|
-
end
|
1815
|
+
purge_peering_connections(noop, vpc.vpc_id, region: region, credentials: credentials)
|
1816
|
+
|
1817
|
+
on_retry = Proc.new {
|
1818
|
+
MU::Cloud.resourceClass("AWS", "FirewallRule").cleanup(
|
1819
|
+
noop: noop,
|
1820
|
+
region: region,
|
1821
|
+
credentials: credentials,
|
1822
|
+
flags: { "vpc_id" => vpc.vpc_id }
|
1823
|
+
)
|
1824
|
+
purge_gateways(noop, tagfilters, region: region, credentials: credentials)
|
2148
1825
|
}
|
2149
1826
|
|
2150
|
-
|
2151
|
-
begin
|
1827
|
+
MU.retrier([Aws::EC2::Errors::DependencyViolation], ignoreme: [Aws::EC2::Errors::InvalidVpcIDNotFound], max: 20, on_retry: on_retry) {
|
2152
1828
|
MU.log "Deleting VPC #{vpc.vpc_id}"
|
2153
1829
|
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_vpc(vpc_id: vpc.vpc_id) if !noop
|
2154
|
-
|
2155
|
-
MU.log "VPC #{vpc.vpc_id} has already been deleted", MU::WARN
|
2156
|
-
rescue Aws::EC2::Errors::DependencyViolation => e
|
2157
|
-
if retries < 5
|
2158
|
-
MU.log "#{vpc.vpc_id} in #{region} had hidden dependencies, will try to remove them", MU::NOTICE
|
2159
|
-
retries += 1
|
2160
|
-
# fry some common rogue resources
|
2161
|
-
MU::Cloud::AWS::FirewallRule.cleanup(
|
2162
|
-
noop: noop,
|
2163
|
-
region: region,
|
2164
|
-
credentials: credentials,
|
2165
|
-
flags: { "vpc_id" => vpc.vpc_id }
|
2166
|
-
)
|
2167
|
-
purge_gateways(noop, tagfilters, region: region, credentials: credentials)
|
2168
|
-
sleep 10
|
2169
|
-
retry
|
2170
|
-
else
|
2171
|
-
MU.log "Failed to remove #{vpc.vpc_id} in #{region}: #{e.message}", MU::ERR
|
2172
|
-
next
|
2173
|
-
end
|
2174
|
-
end
|
1830
|
+
}
|
2175
1831
|
|
2176
1832
|
if !MU::Cloud::AWS.isGovCloud?(region)
|
2177
1833
|
mu_zone = MU::Cloud::DNSZone.find(cloud_id: "platform-mu", region: region, credentials: credentials).values.first
|
2178
1834
|
if !mu_zone.nil?
|
2179
|
-
MU::Cloud
|
1835
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").toggleVPCAccess(id: mu_zone.id, vpc_id: vpc.vpc_id, remove: true, credentials: credentials)
|
2180
1836
|
end
|
2181
1837
|
end
|
2182
1838
|
}
|
2183
1839
|
end
|
2184
1840
|
private_class_method :purge_vpcs
|
2185
1841
|
|
2186
|
-
# Subnets are almost a first-class resource. So let's kinda sorta treat
|
2187
|
-
# them like one. This should only be invoked on objects that already
|
2188
|
-
# exists in the cloud layer.
|
2189
|
-
class Subnet < MU::Cloud::AWS::VPC
|
2190
|
-
|
2191
|
-
attr_reader :cloud_id
|
2192
|
-
attr_reader :ip_block
|
2193
|
-
attr_reader :mu_name
|
2194
|
-
attr_reader :name
|
2195
|
-
attr_reader :az
|
2196
|
-
attr_reader :cloud_desc
|
2197
|
-
|
2198
|
-
# @param parent [MU::Cloud::AWS::VPC]: The parent VPC of this subnet.
|
2199
|
-
# @param config [Hash<String>]:
|
2200
|
-
def initialize(parent, config)
|
2201
|
-
@parent = parent
|
2202
|
-
@config = MU::Config.manxify(config)
|
2203
|
-
@cloud_id = config['cloud_id']
|
2204
|
-
@mu_name = config['mu_name']
|
2205
|
-
@name = config['name']
|
2206
|
-
@deploydata = config # This is a dummy for the sake of describe()
|
2207
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_subnets(subnet_ids: [@cloud_id]).subnets.first
|
2208
|
-
@az = resp.availability_zone
|
2209
|
-
@ip_block = resp.cidr_block
|
2210
|
-
@cloud_desc = resp # XXX this really isn't the cloud implementation's business
|
2211
|
-
|
2212
|
-
end
|
2213
|
-
|
2214
|
-
# Return the cloud identifier for the default route of this subnet.
|
2215
|
-
def defaultRoute
|
2216
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
2217
|
-
filters: [{name: "association.subnet-id", values: [@cloud_id]}]
|
2218
|
-
)
|
2219
|
-
if resp.route_tables.size == 0 # use default route table for the VPC
|
2220
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
2221
|
-
filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
|
2222
|
-
)
|
2223
|
-
end
|
2224
|
-
resp.route_tables.each { |route_table|
|
2225
|
-
route_table.routes.each { |route|
|
2226
|
-
if route.destination_cidr_block =="0.0.0.0/0" and route.state != "blackhole"
|
2227
|
-
return route.instance_id if !route.instance_id.nil?
|
2228
|
-
return route.gateway_id if !route.gateway_id.nil?
|
2229
|
-
return route.vpc_peering_connection_id if !route.vpc_peering_connection_id.nil?
|
2230
|
-
return route.network_interface_id if !route.network_interface_id.nil?
|
2231
|
-
end
|
2232
|
-
}
|
2233
|
-
}
|
2234
|
-
return nil
|
2235
|
-
end
|
2236
|
-
|
2237
|
-
# Is this subnet privately-routable only, or public?
|
2238
|
-
# @return [Boolean]
|
2239
|
-
def private?
|
2240
|
-
return false if @cloud_id.nil?
|
2241
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
2242
|
-
filters: [{name: "association.subnet-id", values: [@cloud_id]}]
|
2243
|
-
)
|
2244
|
-
if resp.route_tables.size == 0 # use default route table for the VPC
|
2245
|
-
resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_route_tables(
|
2246
|
-
filters: [{name: "vpc-id", values: [@parent.cloud_id]}]
|
2247
|
-
)
|
2248
|
-
end
|
2249
|
-
resp.route_tables.each { |route_table|
|
2250
|
-
route_table.routes.each { |route|
|
2251
|
-
return false if !route.gateway_id.nil? and route.gateway_id != "local" # you can have an IgW and route it to a subset of IPs instead of 0.0.0.0/0
|
2252
|
-
if route.destination_cidr_block == "0.0.0.0/0"
|
2253
|
-
return true if !route.instance_id.nil?
|
2254
|
-
return true if route.nat_gateway_id
|
2255
|
-
end
|
2256
|
-
}
|
2257
|
-
}
|
2258
|
-
return true
|
2259
|
-
end
|
2260
|
-
end
|
2261
|
-
|
2262
1842
|
end #class
|
2263
1843
|
end #class
|
2264
1844
|
end
|