lucid-cumulus 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +3 -0
- data/.travis.yml +11 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +29 -0
- data/LICENSE +202 -0
- data/README.md +41 -0
- data/autocomplete +137 -0
- data/bin/cumulus +658 -0
- data/cumulus +2 -0
- data/lib/autoscaling/AutoScaling.rb +40 -0
- data/lib/autoscaling/loader/Loader.rb +56 -0
- data/lib/autoscaling/manager/Manager.rb +360 -0
- data/lib/autoscaling/models/AlarmConfig.rb +165 -0
- data/lib/autoscaling/models/AlarmDiff.rb +172 -0
- data/lib/autoscaling/models/AutoScalingDiff.rb +178 -0
- data/lib/autoscaling/models/GroupConfig.rb +330 -0
- data/lib/autoscaling/models/PolicyConfig.rb +135 -0
- data/lib/autoscaling/models/PolicyDiff.rb +73 -0
- data/lib/autoscaling/models/ScheduledActionDiff.rb +53 -0
- data/lib/autoscaling/models/ScheduledConfig.rb +96 -0
- data/lib/aws_extensions/ec2/DhcpOptions.rb +41 -0
- data/lib/aws_extensions/ec2/Instance.rb +29 -0
- data/lib/aws_extensions/ec2/NetworkAcl.rb +25 -0
- data/lib/aws_extensions/ec2/NetworkInterface.rb +14 -0
- data/lib/aws_extensions/ec2/RouteTable.rb +26 -0
- data/lib/aws_extensions/ec2/SecurityGroup.rb +16 -0
- data/lib/aws_extensions/ec2/Subnet.rb +28 -0
- data/lib/aws_extensions/ec2/Volume.rb +24 -0
- data/lib/aws_extensions/ec2/Vpc.rb +14 -0
- data/lib/aws_extensions/ec2/VpcEndpoint.rb +11 -0
- data/lib/aws_extensions/elb/BackendServerDescription.rb +12 -0
- data/lib/aws_extensions/elb/PolicyDescription.rb +14 -0
- data/lib/aws_extensions/kinesis/StreamDescription.rb +12 -0
- data/lib/aws_extensions/route53/AliasTarget.rb +21 -0
- data/lib/aws_extensions/s3/Bucket.rb +33 -0
- data/lib/aws_extensions/s3/BucketAcl.rb +28 -0
- data/lib/aws_extensions/s3/BucketCors.rb +17 -0
- data/lib/aws_extensions/s3/BucketLifecycle.rb +21 -0
- data/lib/aws_extensions/s3/BucketLogging.rb +18 -0
- data/lib/aws_extensions/s3/BucketNotification.rb +23 -0
- data/lib/aws_extensions/s3/BucketPolicy.rb +18 -0
- data/lib/aws_extensions/s3/BucketTagging.rb +15 -0
- data/lib/aws_extensions/s3/BucketVersioning.rb +14 -0
- data/lib/aws_extensions/s3/BucketWebsite.rb +49 -0
- data/lib/aws_extensions/s3/CORSRule.rb +27 -0
- data/lib/aws_extensions/s3/ReplicationConfiguration.rb +22 -0
- data/lib/cloudfront/CloudFront.rb +83 -0
- data/lib/cloudfront/loader/Loader.rb +31 -0
- data/lib/cloudfront/manager/Manager.rb +183 -0
- data/lib/cloudfront/models/CacheBehaviorConfig.rb +237 -0
- data/lib/cloudfront/models/CacheBehaviorDiff.rb +211 -0
- data/lib/cloudfront/models/CustomOriginConfig.rb +51 -0
- data/lib/cloudfront/models/CustomOriginDiff.rb +74 -0
- data/lib/cloudfront/models/DistributionConfig.rb +183 -0
- data/lib/cloudfront/models/DistributionDiff.rb +131 -0
- data/lib/cloudfront/models/InvalidationConfig.rb +37 -0
- data/lib/cloudfront/models/OriginConfig.rb +144 -0
- data/lib/cloudfront/models/OriginDiff.rb +86 -0
- data/lib/cloudfront/models/OriginSslProtocols.rb +28 -0
- data/lib/cloudfront/models/OriginSslProtocolsDiff.rb +39 -0
- data/lib/common/BaseLoader.rb +80 -0
- data/lib/common/manager/Manager.rb +148 -0
- data/lib/common/models/Diff.rb +114 -0
- data/lib/common/models/ListChange.rb +21 -0
- data/lib/common/models/TagsDiff.rb +55 -0
- data/lib/common/models/UTCTimeSource.rb +17 -0
- data/lib/conf/Configuration.rb +365 -0
- data/lib/ec2/EC2.rb +503 -0
- data/lib/ec2/IPProtocolMapping.rb +165 -0
- data/lib/ec2/loaders/EbsLoader.rb +19 -0
- data/lib/ec2/loaders/InstanceLoader.rb +32 -0
- data/lib/ec2/managers/EbsManager.rb +176 -0
- data/lib/ec2/managers/InstanceManager.rb +509 -0
- data/lib/ec2/models/EbsGroupConfig.rb +133 -0
- data/lib/ec2/models/EbsGroupDiff.rb +48 -0
- data/lib/ec2/models/InstanceConfig.rb +202 -0
- data/lib/ec2/models/InstanceDiff.rb +95 -0
- data/lib/elb/ELB.rb +148 -0
- data/lib/elb/loader/Loader.rb +65 -0
- data/lib/elb/manager/Manager.rb +581 -0
- data/lib/elb/models/AccessLogConfig.rb +82 -0
- data/lib/elb/models/AccessLogDiff.rb +47 -0
- data/lib/elb/models/HealthCheckConfig.rb +91 -0
- data/lib/elb/models/HealthCheckDiff.rb +50 -0
- data/lib/elb/models/ListenerConfig.rb +99 -0
- data/lib/elb/models/ListenerDiff.rb +91 -0
- data/lib/elb/models/LoadBalancerConfig.rb +239 -0
- data/lib/elb/models/LoadBalancerDiff.rb +265 -0
- data/lib/iam/IAM.rb +36 -0
- data/lib/iam/loader/Loader.rb +117 -0
- data/lib/iam/manager/IamGroups.rb +98 -0
- data/lib/iam/manager/IamResource.rb +288 -0
- data/lib/iam/manager/IamRoles.rb +112 -0
- data/lib/iam/manager/IamUsers.rb +54 -0
- data/lib/iam/manager/Manager.rb +29 -0
- data/lib/iam/migration/AssumeRoleUnifier.rb +34 -0
- data/lib/iam/migration/PolicyUnifier.rb +90 -0
- data/lib/iam/models/GroupConfig.rb +40 -0
- data/lib/iam/models/IamDiff.rb +132 -0
- data/lib/iam/models/PolicyConfig.rb +67 -0
- data/lib/iam/models/ResourceWithPolicy.rb +208 -0
- data/lib/iam/models/RoleConfig.rb +53 -0
- data/lib/iam/models/StatementConfig.rb +35 -0
- data/lib/iam/models/UserConfig.rb +21 -0
- data/lib/kinesis/Kinesis.rb +94 -0
- data/lib/kinesis/loader/Loader.rb +19 -0
- data/lib/kinesis/manager/Manager.rb +206 -0
- data/lib/kinesis/models/StreamConfig.rb +75 -0
- data/lib/kinesis/models/StreamDiff.rb +58 -0
- data/lib/lambda/Lambda.rb +41 -0
- data/lib/route53/loader/Loader.rb +32 -0
- data/lib/route53/manager/Manager.rb +241 -0
- data/lib/route53/models/AliasTarget.rb +86 -0
- data/lib/route53/models/RecordConfig.rb +178 -0
- data/lib/route53/models/RecordDiff.rb +140 -0
- data/lib/route53/models/Vpc.rb +24 -0
- data/lib/route53/models/ZoneConfig.rb +156 -0
- data/lib/route53/models/ZoneDiff.rb +118 -0
- data/lib/s3/S3.rb +89 -0
- data/lib/s3/loader/Loader.rb +66 -0
- data/lib/s3/manager/Manager.rb +296 -0
- data/lib/s3/models/BucketConfig.rb +321 -0
- data/lib/s3/models/BucketDiff.rb +167 -0
- data/lib/s3/models/GrantConfig.rb +189 -0
- data/lib/s3/models/GrantDiff.rb +50 -0
- data/lib/s3/models/LifecycleConfig.rb +142 -0
- data/lib/s3/models/LifecycleDiff.rb +46 -0
- data/lib/s3/models/LoggingConfig.rb +81 -0
- data/lib/s3/models/NotificationConfig.rb +157 -0
- data/lib/s3/models/NotificationDiff.rb +62 -0
- data/lib/s3/models/ReplicationConfig.rb +133 -0
- data/lib/s3/models/ReplicationDiff.rb +60 -0
- data/lib/s3/models/WebsiteConfig.rb +107 -0
- data/lib/security/SecurityGroups.rb +39 -0
- data/lib/security/loader/Loader.rb +94 -0
- data/lib/security/manager/Manager.rb +246 -0
- data/lib/security/models/RuleConfig.rb +161 -0
- data/lib/security/models/RuleDiff.rb +72 -0
- data/lib/security/models/RuleMigration.rb +127 -0
- data/lib/security/models/SecurityGroupConfig.rb +172 -0
- data/lib/security/models/SecurityGroupDiff.rb +112 -0
- data/lib/sns/SNS.rb +40 -0
- data/lib/sqs/SQS.rb +62 -0
- data/lib/sqs/loader/Loader.rb +34 -0
- data/lib/sqs/manager/Manager.rb +128 -0
- data/lib/sqs/models/DeadLetterConfig.rb +70 -0
- data/lib/sqs/models/DeadLetterDiff.rb +35 -0
- data/lib/sqs/models/QueueConfig.rb +115 -0
- data/lib/sqs/models/QueueDiff.rb +89 -0
- data/lib/util/Colors.rb +111 -0
- data/lib/util/StatusCodes.rb +51 -0
- data/lib/vpc/loader/Loader.rb +73 -0
- data/lib/vpc/manager/Manager.rb +954 -0
- data/lib/vpc/models/AclEntryConfig.rb +150 -0
- data/lib/vpc/models/AclEntryDiff.rb +54 -0
- data/lib/vpc/models/DhcpConfig.rb +100 -0
- data/lib/vpc/models/DhcpDiff.rb +90 -0
- data/lib/vpc/models/EndpointConfig.rb +76 -0
- data/lib/vpc/models/EndpointDiff.rb +69 -0
- data/lib/vpc/models/NetworkAclConfig.rb +87 -0
- data/lib/vpc/models/NetworkAclDiff.rb +116 -0
- data/lib/vpc/models/RouteConfig.rb +82 -0
- data/lib/vpc/models/RouteDiff.rb +50 -0
- data/lib/vpc/models/RouteTableConfig.rb +92 -0
- data/lib/vpc/models/RouteTableDiff.rb +101 -0
- data/lib/vpc/models/SubnetConfig.rb +113 -0
- data/lib/vpc/models/SubnetDiff.rb +78 -0
- data/lib/vpc/models/VpcConfig.rb +173 -0
- data/lib/vpc/models/VpcDiff.rb +315 -0
- data/lucid-cumulus.gemspec +20 -0
- data/rakefile.rb +8 -0
- metadata +245 -0
@@ -0,0 +1,19 @@
|
|
1
|
+
require "common/BaseLoader"
|
2
|
+
require "conf/Configuration"
|
3
|
+
require "kinesis/models/StreamConfig"
|
4
|
+
|
5
|
+
module Cumulus
|
6
|
+
module Kinesis
|
7
|
+
module Loader
|
8
|
+
|
9
|
+
include Common::BaseLoader
|
10
|
+
|
11
|
+
@@streams_dir = Configuration.instance.kinesis.directory
|
12
|
+
|
13
|
+
def self.streams
|
14
|
+
Common::BaseLoader::resources(@@streams_dir, &StreamConfig.method(:new))
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,206 @@
|
|
1
|
+
require "common/manager/Manager"
|
2
|
+
require "conf/Configuration"
|
3
|
+
require "kinesis/Kinesis"
|
4
|
+
require "kinesis/loader/Loader"
|
5
|
+
require "kinesis/models/StreamConfig"
|
6
|
+
require "kinesis/models/StreamDiff"
|
7
|
+
|
8
|
+
require "aws-sdk"
|
9
|
+
|
10
|
+
module Cumulus
|
11
|
+
module Kinesis
|
12
|
+
class Manager < Common::Manager
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
super()
|
16
|
+
@create_asset = true
|
17
|
+
@client = Aws::Kinesis::Client.new(Configuration.instance.client)
|
18
|
+
end
|
19
|
+
|
20
|
+
def resource_name
|
21
|
+
"Kinesis Stream"
|
22
|
+
end
|
23
|
+
|
24
|
+
def local_resources
|
25
|
+
@local_resources ||= Hash[Loader.streams.map { |local| [local.name, local] }]
|
26
|
+
end
|
27
|
+
|
28
|
+
def aws_resources
|
29
|
+
@aws_resources ||= Kinesis::named_streams
|
30
|
+
end
|
31
|
+
|
32
|
+
def unmanaged_diff(aws)
|
33
|
+
StreamDiff.unmanaged(aws)
|
34
|
+
end
|
35
|
+
|
36
|
+
def added_diff(local)
|
37
|
+
StreamDiff.added(local)
|
38
|
+
end
|
39
|
+
|
40
|
+
def diff_resource(local, aws)
|
41
|
+
local.diff(aws)
|
42
|
+
end
|
43
|
+
|
44
|
+
def migrate
|
45
|
+
puts Colors.blue("Migrating Kinesis Streams...")
|
46
|
+
|
47
|
+
# Create the directories
|
48
|
+
streams_dir = "#{@migration_root}/kinesis"
|
49
|
+
|
50
|
+
if !Dir.exists?(@migration_root)
|
51
|
+
Dir.mkdir(@migration_root)
|
52
|
+
end
|
53
|
+
if !Dir.exists?(streams_dir)
|
54
|
+
Dir.mkdir(streams_dir)
|
55
|
+
end
|
56
|
+
|
57
|
+
Kinesis::named_streams.each do |name, stream|
|
58
|
+
puts "Migrating stream #{name}"
|
59
|
+
|
60
|
+
cumulus_stream = StreamConfig.new(name).populate!(stream)
|
61
|
+
json = JSON.pretty_generate(cumulus_stream.to_hash)
|
62
|
+
File.open("#{streams_dir}/#{name}.json", "w") { |f| f.write(json) }
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
def create(local)
|
68
|
+
@client.create_stream({
|
69
|
+
stream_name: local.name,
|
70
|
+
shard_count: local.shards
|
71
|
+
})
|
72
|
+
|
73
|
+
@client.wait_until(:stream_exists, {
|
74
|
+
stream_name: local.name
|
75
|
+
})
|
76
|
+
|
77
|
+
# Describe the newly created stream
|
78
|
+
created_stream = Kinesis::describe_stream(local.name)
|
79
|
+
|
80
|
+
# If the stream retention period is different, then update it
|
81
|
+
if created_stream.retention_period_hours > local.retention_period
|
82
|
+
@client.decrease_stream_retention_period({
|
83
|
+
stream_name: local.name,
|
84
|
+
retention_period_hours: local.retention_period
|
85
|
+
})
|
86
|
+
elsif created_stream.retention_period_hours < local.retention_period
|
87
|
+
@client.increase_stream_retention_period({
|
88
|
+
stream_name: local.name,
|
89
|
+
retention_period_hours: local.retention_period
|
90
|
+
})
|
91
|
+
end
|
92
|
+
|
93
|
+
# If the created stream has tags, add them
|
94
|
+
if !local.tags.empty?
|
95
|
+
@client.add_tags_to_stream({
|
96
|
+
stream_name: local.name,
|
97
|
+
tags: local.tags
|
98
|
+
})
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
def update(local, diffs)
|
104
|
+
diffs.each do |diff|
|
105
|
+
case diff.type
|
106
|
+
when StreamChange::SHARDS
|
107
|
+
|
108
|
+
# See if we are splitting or merging and make sure it is a multiple of 2
|
109
|
+
if diff.aws < diff.local
|
110
|
+
if diff.aws != diff.local / 2.0
|
111
|
+
puts Colors.red("Can only increase the number of shards by a factor of 2")
|
112
|
+
else
|
113
|
+
aws_stream = Kinesis::named_streams[local.name]
|
114
|
+
|
115
|
+
# Split the shards 1 at a time
|
116
|
+
aws_stream.sorted_shards.each do |shard|
|
117
|
+
puts Colors.blue("Splitting shard #{shard.shard_id}")
|
118
|
+
|
119
|
+
# The splitting point is halfway between the hash start and end
|
120
|
+
hash_start = shard.hash_key_range.starting_hash_key.to_i
|
121
|
+
hash_end = shard.hash_key_range.ending_hash_key.to_i
|
122
|
+
hash_split = hash_start + ((hash_end - hash_start) / 2)
|
123
|
+
|
124
|
+
@client.split_shard({
|
125
|
+
stream_name: local.name,
|
126
|
+
shard_to_split: shard.shard_id,
|
127
|
+
new_starting_hash_key: hash_split.to_s
|
128
|
+
})
|
129
|
+
|
130
|
+
# After every split we have to wait until the stream is ready
|
131
|
+
@client.wait_until(:stream_exists, {
|
132
|
+
stream_name: local.name
|
133
|
+
})
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
elsif diff.aws > diff.local
|
138
|
+
aws_stream = Kinesis::named_streams[local.name]
|
139
|
+
|
140
|
+
if aws_stream.sorted_shards.length != local.shards * 2.0
|
141
|
+
puts Colors.red("Can only decrease the number of shards by a factor of 2")
|
142
|
+
else
|
143
|
+
# Merge the sorted shards in groups of 2
|
144
|
+
aws_stream.sorted_shards.each_slice(2) do |slice|
|
145
|
+
puts Colors.blue("Merging shards #{slice[0].shard_id} and #{slice[1].shard_id}")
|
146
|
+
|
147
|
+
@client.merge_shards({
|
148
|
+
stream_name: local.name,
|
149
|
+
shard_to_merge: slice[0].shard_id,
|
150
|
+
adjacent_shard_to_merge: slice[1].shard_id
|
151
|
+
})
|
152
|
+
|
153
|
+
# After every merge we have to wait until the stream is ready
|
154
|
+
@client.wait_until(:stream_exists, {
|
155
|
+
stream_name: local.name
|
156
|
+
})
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
when StreamChange::RETENTION
|
163
|
+
puts Colors.blue("Updating retention period...")
|
164
|
+
|
165
|
+
|
166
|
+
if diff.aws > local.retention_period
|
167
|
+
@client.decrease_stream_retention_period({
|
168
|
+
stream_name: local.name,
|
169
|
+
retention_period_hours: local.retention_period
|
170
|
+
})
|
171
|
+
elsif diff.aws < local.retention_period
|
172
|
+
@client.increase_stream_retention_period({
|
173
|
+
stream_name: local.name,
|
174
|
+
retention_period_hours: local.retention_period
|
175
|
+
})
|
176
|
+
end
|
177
|
+
|
178
|
+
# Wait for the stream to be in an active state or shard updates will fail
|
179
|
+
@client.wait_until(:stream_exists, {
|
180
|
+
stream_name: local.name
|
181
|
+
})
|
182
|
+
when StreamChange::TAGS
|
183
|
+
puts Colors.blue("Updating tags...")
|
184
|
+
|
185
|
+
if !diff.tags_to_remove.empty?
|
186
|
+
@client.remove_tags_from_stream({
|
187
|
+
stream_name: local.name,
|
188
|
+
tag_keys: diff.tags_to_remove.keys
|
189
|
+
})
|
190
|
+
end
|
191
|
+
|
192
|
+
if !diff.tags_to_add.empty?
|
193
|
+
@client.add_tags_to_stream({
|
194
|
+
stream_name: local.name,
|
195
|
+
tags: diff.tags_to_add
|
196
|
+
})
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require "conf/Configuration"
|
2
|
+
require "kinesis/loader/Loader"
|
3
|
+
require "kinesis/models/StreamDiff"
|
4
|
+
|
5
|
+
require "json"
|
6
|
+
|
7
|
+
module Cumulus
|
8
|
+
module Kinesis
|
9
|
+
|
10
|
+
# Public: An object representing configuration for a Kiensis stream
|
11
|
+
class StreamConfig
|
12
|
+
attr_reader :name
|
13
|
+
attr_reader :retention_period
|
14
|
+
attr_reader :shards
|
15
|
+
attr_reader :tags
|
16
|
+
|
17
|
+
# Public: Constructor
|
18
|
+
#
|
19
|
+
# json - a hash containing the JSON configuration for the stream
|
20
|
+
def initialize(name, json = nil)
|
21
|
+
@name = name
|
22
|
+
if !json.nil?
|
23
|
+
@shards = json["shards"]
|
24
|
+
@retention_period = json["retention-period"] || 24
|
25
|
+
@tags = json["tags"] || {}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def to_hash
|
30
|
+
{
|
31
|
+
"retention-period" => @retention_period,
|
32
|
+
"shards" => @shards,
|
33
|
+
"tags" => @tags
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
# Public: Populate a config object with AWS configuration
|
38
|
+
#
|
39
|
+
# aws - the AWS configuration for the strean
|
40
|
+
def populate!(aws)
|
41
|
+
@retention_period = aws.retention_period_hours
|
42
|
+
@shards = aws.sorted_shards.length
|
43
|
+
@tags = Kinesis::stream_tags[aws.stream_name] || {}
|
44
|
+
|
45
|
+
self
|
46
|
+
end
|
47
|
+
|
48
|
+
# Public: Produce an array of differences between this local configuration and the
|
49
|
+
# configuration in AWS
|
50
|
+
#
|
51
|
+
# aws - the AWS resource
|
52
|
+
#
|
53
|
+
# Returns an array of the StreamDiffs that were found
|
54
|
+
def diff(aws)
|
55
|
+
diffs = []
|
56
|
+
|
57
|
+
if @retention_period != aws.retention_period_hours
|
58
|
+
diffs << StreamDiff.new(StreamChange::RETENTION, aws.retention_period_hours, @retention_period)
|
59
|
+
end
|
60
|
+
|
61
|
+
if @shards != aws.sorted_shards.length
|
62
|
+
diffs << StreamDiff.new(StreamChange::SHARDS, aws.sorted_shards.length, @shards)
|
63
|
+
end
|
64
|
+
|
65
|
+
aws_tags = Kinesis::stream_tags[aws.stream_name]
|
66
|
+
if @tags != aws_tags
|
67
|
+
diffs << StreamDiff.new(StreamChange::TAGS, aws_tags, @tags)
|
68
|
+
end
|
69
|
+
|
70
|
+
diffs
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require "common/models/Diff"
|
2
|
+
require "common/models/ListChange"
|
3
|
+
require "common/models/TagsDiff"
|
4
|
+
require "util/Colors"
|
5
|
+
|
6
|
+
module Cumulus
|
7
|
+
module Kinesis
|
8
|
+
# Public: The types of changes that can be made to a stream
|
9
|
+
module StreamChange
|
10
|
+
include Common::DiffChange
|
11
|
+
|
12
|
+
SHARDS = Common::DiffChange.next_change_id
|
13
|
+
RETENTION = Common::DiffChange.next_change_id
|
14
|
+
TAGS = Common::DiffChange.next_change_id
|
15
|
+
end
|
16
|
+
|
17
|
+
# Public: Represents a single difference between local configuration and AWS configuration
|
18
|
+
class StreamDiff < Common::Diff
|
19
|
+
include StreamChange
|
20
|
+
include Common::TagsDiff
|
21
|
+
|
22
|
+
def local_tags
|
23
|
+
@local
|
24
|
+
end
|
25
|
+
|
26
|
+
def aws_tags
|
27
|
+
@aws
|
28
|
+
end
|
29
|
+
|
30
|
+
def asset_type
|
31
|
+
"Stream"
|
32
|
+
end
|
33
|
+
|
34
|
+
def aws_name
|
35
|
+
@aws.stream_name
|
36
|
+
end
|
37
|
+
|
38
|
+
def diff_string
|
39
|
+
case @type
|
40
|
+
when SHARDS
|
41
|
+
[
|
42
|
+
"Shards:",
|
43
|
+
Colors.aws_changes("\tAWS - #{aws}"),
|
44
|
+
Colors.local_changes("\tLocal - #{local}"),
|
45
|
+
].join("\n")
|
46
|
+
when RETENTION
|
47
|
+
[
|
48
|
+
"Retention:",
|
49
|
+
Colors.aws_changes("\tAWS - #{aws} hours"),
|
50
|
+
Colors.local_changes("\tLocal - #{local} hours"),
|
51
|
+
].join("\n")
|
52
|
+
when TAGS
|
53
|
+
tags_diff_string
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require "conf/Configuration"
|
2
|
+
|
3
|
+
require "aws-sdk"
|
4
|
+
|
5
|
+
module Cumulus
|
6
|
+
module Lambda
|
7
|
+
class << self
|
8
|
+
@@client = Aws::Lambda::Client.new(Configuration.instance.client)
|
9
|
+
|
10
|
+
# Public: Static method that will get a Lambda function from AWS by its
|
11
|
+
# name
|
12
|
+
#
|
13
|
+
# name - the name of the function to get
|
14
|
+
#
|
15
|
+
# Returns the function
|
16
|
+
def get_aws(name)
|
17
|
+
functions.fetch(name)
|
18
|
+
rescue KeyError
|
19
|
+
puts "No Lambda function named #{name}"
|
20
|
+
exit
|
21
|
+
end
|
22
|
+
|
23
|
+
# Public: Provide a mapping of functions to their names. Lazily loads
|
24
|
+
# resources.
|
25
|
+
#
|
26
|
+
# Returns the functions mapped to their names
|
27
|
+
def functions
|
28
|
+
@functions ||= init_functions
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
# Internal: Load the functions and map them to their names
|
34
|
+
#
|
35
|
+
# Returns the functions mapped to their names
|
36
|
+
def init_functions
|
37
|
+
Hash[@@client.list_functions.functions.map { |f| [f.function_name, f] }]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require "common/BaseLoader"
|
2
|
+
require "conf/Configuration"
|
3
|
+
require "route53/models/ZoneConfig"
|
4
|
+
|
5
|
+
# Public: Load Route53 assets
|
6
|
+
module Cumulus
|
7
|
+
module Route53
|
8
|
+
module Loader
|
9
|
+
include Common::BaseLoader
|
10
|
+
|
11
|
+
@@zones_dir = Configuration.instance.route53.zones_directory
|
12
|
+
@@includes_dir = Configuration.instance.route53.includes_directory
|
13
|
+
|
14
|
+
# Public: Load all the zone configurations as ZoneConfig objects
|
15
|
+
#
|
16
|
+
# Returns an array of ZoneConfig
|
17
|
+
def self.zones
|
18
|
+
Common::BaseLoader::resources(@@zones_dir, &ZoneConfig.method(:new))
|
19
|
+
end
|
20
|
+
|
21
|
+
# Public: Load a single "includes file" as parsed JSON
|
22
|
+
#
|
23
|
+
# name - the name of the file to include
|
24
|
+
#
|
25
|
+
# Returns an array of parsed JSON
|
26
|
+
def self.includes_file(name)
|
27
|
+
Common::BaseLoader::resource(name, @@includes_dir, &Proc.new { |n, json| json })
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,241 @@
|
|
1
|
+
require "common/manager/Manager"
|
2
|
+
require "conf/Configuration"
|
3
|
+
require "route53/loader/Loader"
|
4
|
+
require "route53/models/RecordDiff"
|
5
|
+
require "route53/models/Vpc"
|
6
|
+
require "route53/models/ZoneDiff"
|
7
|
+
require "util/Colors"
|
8
|
+
|
9
|
+
require "aws-sdk"
|
10
|
+
|
11
|
+
module Cumulus
|
12
|
+
module Route53
|
13
|
+
class Manager < Common::Manager
|
14
|
+
def initialize
|
15
|
+
super()
|
16
|
+
@create_asset = false
|
17
|
+
@route53 = Aws::Route53::Client.new(Configuration.instance.client)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Public: Migrate AWS Route53 configuration to Cumulus configuration.
|
21
|
+
def migrate
|
22
|
+
zones_dir = "#{@migration_root}/zones"
|
23
|
+
|
24
|
+
if !Dir.exists?(@migration_root)
|
25
|
+
Dir.mkdir(@migration_root)
|
26
|
+
end
|
27
|
+
if !Dir.exists?(zones_dir)
|
28
|
+
Dir.mkdir(zones_dir)
|
29
|
+
end
|
30
|
+
|
31
|
+
aws_resources.each_value do |resource|
|
32
|
+
puts "Processing #{resource.name}..."
|
33
|
+
config = ZoneConfig.new(resource.name)
|
34
|
+
config.populate(resource)
|
35
|
+
|
36
|
+
puts "Writing #{resource.name} configuration to file"
|
37
|
+
filename = if config.private then "#{config.name}-private" else config.name end
|
38
|
+
File.open("#{zones_dir}/#{filename.sub(".", "-")}.json", "w") { |f| f.write(config.pretty_json) }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def resource_name
|
43
|
+
"Zone"
|
44
|
+
end
|
45
|
+
|
46
|
+
def local_resources
|
47
|
+
@local_resources ||= Hash[Loader.zones.map { |local| [local.id, local] }]
|
48
|
+
end
|
49
|
+
|
50
|
+
def aws_resources
|
51
|
+
@aws_resources ||= init_aws_resources
|
52
|
+
end
|
53
|
+
|
54
|
+
def unmanaged_diff(aws)
|
55
|
+
ZoneDiff.unmanaged(aws)
|
56
|
+
end
|
57
|
+
|
58
|
+
def added_diff(local)
|
59
|
+
ZoneDiff.added(local)
|
60
|
+
end
|
61
|
+
|
62
|
+
def diff_resource(local, aws)
|
63
|
+
local.diff(aws)
|
64
|
+
end
|
65
|
+
|
66
|
+
def update(local, diffs)
|
67
|
+
diffs.each do |diff|
|
68
|
+
case diff.type
|
69
|
+
when ZoneChange::COMMENT
|
70
|
+
puts Colors.blue("\tupdating comment...")
|
71
|
+
update_comment(local.id, local.comment)
|
72
|
+
when ZoneChange::DOMAIN
|
73
|
+
puts "\tAWS doesn't allow you to change the domain for a zone."
|
74
|
+
when ZoneChange::PRIVATE
|
75
|
+
puts "\tAWS doesn't allow you to change whether a zone is private."
|
76
|
+
when ZoneChange::VPC
|
77
|
+
update_vpc(local.id, diff.added_vpc_ids, diff.removed_vpc_ids)
|
78
|
+
when ZoneChange::RECORD
|
79
|
+
update_records(
|
80
|
+
local.id,
|
81
|
+
diff.changed_records.reject do |r|
|
82
|
+
r.type == RecordChange::IGNORED or r.type == RecordChange::DEFAULT
|
83
|
+
end
|
84
|
+
)
|
85
|
+
|
86
|
+
ignored = diff.changed_records.select { |r| r.type == RecordChange::IGNORED }
|
87
|
+
if Configuration.instance.route53.print_all_ignored
|
88
|
+
ignored.each do |record_diff|
|
89
|
+
puts "\tIgnoring record #{record_diff.aws_name}"
|
90
|
+
end
|
91
|
+
else
|
92
|
+
if ignored.size > 0
|
93
|
+
puts "\tYour blacklist ignored #{ignored.size} records."
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
private
|
101
|
+
|
102
|
+
# Internal: Update the comment associated with a zone.
|
103
|
+
#
|
104
|
+
# id - the id of the zone to update
|
105
|
+
# comment - the new comment
|
106
|
+
def update_comment(id, comment)
|
107
|
+
@route53.update_hosted_zone_comment({
|
108
|
+
id: id,
|
109
|
+
comment: comment
|
110
|
+
})
|
111
|
+
end
|
112
|
+
|
113
|
+
# Internal: Update the VPCs associated with a zone.
|
114
|
+
#
|
115
|
+
# id - the id of the zone to update
|
116
|
+
# associate - the vpc ids to associate with the zone
|
117
|
+
# dissociate - the vpc ids to dissociate from the zone
|
118
|
+
def update_vpc(id, associate, dissociate)
|
119
|
+
if !associate.empty?
|
120
|
+
puts Colors.blue("\tassociating VPCs...")
|
121
|
+
associate.each do |vpc|
|
122
|
+
@route53.associate_vpc_with_hosted_zone({
|
123
|
+
hosted_zone_id: id,
|
124
|
+
vpc: { vpc_id: vpc.id, vpc_region: vpc.region }
|
125
|
+
})
|
126
|
+
end
|
127
|
+
end
|
128
|
+
if !dissociate.empty?
|
129
|
+
puts Colors.blue("\tdissociating VPCs...")
|
130
|
+
dissociate.each do |vpc|
|
131
|
+
@route53.disassociate_vpc_from_hosted_zone({
|
132
|
+
hosted_zone_id: id,
|
133
|
+
vpc: { vpc_id: vpc.id, vpc_region: vpc.region }
|
134
|
+
})
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
# Internal: Update the records associated with a zone.
|
140
|
+
#
|
141
|
+
# id - the id of the zone to update
|
142
|
+
# records - RecordDiff objects representing the changes
|
143
|
+
def update_records(id, records)
|
144
|
+
puts Colors.blue("\tupdating records...")
|
145
|
+
if !records.empty?
|
146
|
+
changes = records.map do |record|
|
147
|
+
action = nil
|
148
|
+
resource = nil
|
149
|
+
|
150
|
+
case record.type
|
151
|
+
when RecordChange::CHANGED
|
152
|
+
action = "UPSERT"
|
153
|
+
resource = record.local
|
154
|
+
when RecordChange::ADD
|
155
|
+
action = "CREATE"
|
156
|
+
resource = record.local
|
157
|
+
when RecordChange::UNMANAGED
|
158
|
+
action = "DELETE"
|
159
|
+
resource = record.aws
|
160
|
+
end
|
161
|
+
|
162
|
+
{
|
163
|
+
action: action,
|
164
|
+
resource_record_set: {
|
165
|
+
name: resource.name.gsub("@", "\\\\100"),
|
166
|
+
type: resource.type,
|
167
|
+
ttl: resource.ttl,
|
168
|
+
resource_records: resource.resource_records,
|
169
|
+
alias_target: if resource.alias_target.nil? then nil else {
|
170
|
+
hosted_zone_id: resource.alias_target.hosted_zone_id,
|
171
|
+
dns_name: resource.alias_target.dns_name,
|
172
|
+
evaluate_target_health: resource.alias_target.evaluate_target_health
|
173
|
+
} end
|
174
|
+
}
|
175
|
+
}
|
176
|
+
end
|
177
|
+
|
178
|
+
@route53.change_resource_record_sets({
|
179
|
+
hosted_zone_id: id,
|
180
|
+
change_batch: {
|
181
|
+
changes: changes
|
182
|
+
}
|
183
|
+
})
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# A struct that combines all the data about a hosted zone in AWS
|
188
|
+
AwsZone = Struct.new(:id, :name, :config, :vpc, :route53) do
|
189
|
+
def records
|
190
|
+
@records ||= get_zone_records
|
191
|
+
end
|
192
|
+
|
193
|
+
private
|
194
|
+
|
195
|
+
# Internal: Get the records for this hosted zone.
|
196
|
+
#
|
197
|
+
# Returns an array of records belonging to the zone
|
198
|
+
def get_zone_records()
|
199
|
+
records = []
|
200
|
+
all_records_retrieved = false
|
201
|
+
next_record_name = nil
|
202
|
+
next_record_type = nil
|
203
|
+
next_record_identifier = nil
|
204
|
+
|
205
|
+
until all_records_retrieved
|
206
|
+
response = route53.list_resource_record_sets({
|
207
|
+
hosted_zone_id: id,
|
208
|
+
start_record_name: next_record_name,
|
209
|
+
start_record_type: next_record_type,
|
210
|
+
start_record_identitifier: next_record_identifier
|
211
|
+
}.reject { |k, v| v.nil? })
|
212
|
+
records << response.resource_record_sets
|
213
|
+
next_record_name = response.next_record_name
|
214
|
+
next_record_type = response.next_record_type
|
215
|
+
next_record_identifier = response.next_record_identifier
|
216
|
+
|
217
|
+
if !response.is_truncated
|
218
|
+
all_records_retrieved = true
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
records.flatten.map { |r| r.name = r.name.chomp(".").sub(/\\052/, "*"); r }
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
226
|
+
def init_aws_resources
|
227
|
+
aws = @route53.list_hosted_zones.hosted_zones.map do |zone|
|
228
|
+
vpc = if zone.config.private_zone
|
229
|
+
details = @route53.get_hosted_zone(id: zone.id)
|
230
|
+
details.vp_cs.map { |v| Vpc.new(v.vpc_id, v.vpc_region) }
|
231
|
+
else
|
232
|
+
nil
|
233
|
+
end
|
234
|
+
AwsZone.new(zone.id, zone.name.chomp("."), zone.config, vpc, @route53)
|
235
|
+
end
|
236
|
+
Hash[aws.map { |z| [z.id, z] }]
|
237
|
+
end
|
238
|
+
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|