simplygenius-atmos 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/LICENSE +13 -0
- data/README.md +212 -0
- data/exe/atmos +4 -0
- data/exe/atmos-docker +12 -0
- data/lib/atmos.rb +12 -0
- data/lib/atmos/cli.rb +105 -0
- data/lib/atmos/commands/account.rb +65 -0
- data/lib/atmos/commands/apply.rb +20 -0
- data/lib/atmos/commands/auth_exec.rb +29 -0
- data/lib/atmos/commands/base_command.rb +12 -0
- data/lib/atmos/commands/bootstrap.rb +72 -0
- data/lib/atmos/commands/container.rb +58 -0
- data/lib/atmos/commands/destroy.rb +18 -0
- data/lib/atmos/commands/generate.rb +90 -0
- data/lib/atmos/commands/init.rb +18 -0
- data/lib/atmos/commands/new.rb +18 -0
- data/lib/atmos/commands/otp.rb +54 -0
- data/lib/atmos/commands/plan.rb +20 -0
- data/lib/atmos/commands/secret.rb +87 -0
- data/lib/atmos/commands/terraform.rb +52 -0
- data/lib/atmos/commands/user.rb +74 -0
- data/lib/atmos/config.rb +208 -0
- data/lib/atmos/exceptions.rb +9 -0
- data/lib/atmos/generator.rb +199 -0
- data/lib/atmos/generator_factory.rb +93 -0
- data/lib/atmos/ipc.rb +132 -0
- data/lib/atmos/ipc_actions/notify.rb +27 -0
- data/lib/atmos/ipc_actions/ping.rb +19 -0
- data/lib/atmos/logging.rb +160 -0
- data/lib/atmos/otp.rb +61 -0
- data/lib/atmos/provider_factory.rb +19 -0
- data/lib/atmos/providers/aws/account_manager.rb +82 -0
- data/lib/atmos/providers/aws/auth_manager.rb +208 -0
- data/lib/atmos/providers/aws/container_manager.rb +116 -0
- data/lib/atmos/providers/aws/provider.rb +51 -0
- data/lib/atmos/providers/aws/s3_secret_manager.rb +49 -0
- data/lib/atmos/providers/aws/user_manager.rb +211 -0
- data/lib/atmos/settings_hash.rb +90 -0
- data/lib/atmos/terraform_executor.rb +267 -0
- data/lib/atmos/ui.rb +159 -0
- data/lib/atmos/utils.rb +50 -0
- data/lib/atmos/version.rb +3 -0
- data/templates/new/config/atmos.yml +50 -0
- data/templates/new/config/atmos/runtime.yml +43 -0
- data/templates/new/templates.yml +1 -0
- metadata +526 -0
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../atmos'
|
2
|
+
|
3
|
+
module Atmos
|
4
|
+
class ProviderFactory
|
5
|
+
include GemLogger::LoggerSupport
|
6
|
+
|
7
|
+
def self.get(name)
|
8
|
+
@provider ||= begin
|
9
|
+
logger.debug("Loading provider: #{name}")
|
10
|
+
require "atmos/providers/#{name}/provider"
|
11
|
+
provider = "Atmos::Providers::#{name.camelize}::Provider".constantize
|
12
|
+
logger.debug("Loaded provider #{provider}")
|
13
|
+
provider.new(name)
|
14
|
+
end
|
15
|
+
return @provider
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require_relative '../../../atmos'
|
2
|
+
require 'aws-sdk-organizations'
|
3
|
+
|
4
|
+
module Atmos
|
5
|
+
module Providers
|
6
|
+
module Aws
|
7
|
+
|
8
|
+
class AccountManager
|
9
|
+
include GemLogger::LoggerSupport
|
10
|
+
include FileUtils
|
11
|
+
|
12
|
+
def initialize(provider)
|
13
|
+
@provider = provider
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_account(env, name: nil, email: nil)
|
17
|
+
result = {}
|
18
|
+
org = ::Aws::Organizations::Client.new
|
19
|
+
resp = nil
|
20
|
+
name ||= "Atmos #{env} account"
|
21
|
+
|
22
|
+
begin
|
23
|
+
logger.info "Looking up organization"
|
24
|
+
resp = org.describe_organization()
|
25
|
+
logger.debug "Described organization: #{resp.to_h}"
|
26
|
+
rescue ::Aws::Organizations::Errors::AWSOrganizationsNotInUseException
|
27
|
+
logger.info "Organization doesn't exist, creating"
|
28
|
+
resp = org.create_organization()
|
29
|
+
logger.debug "Created organization: #{resp.to_h}"
|
30
|
+
end
|
31
|
+
|
32
|
+
if email.blank?
|
33
|
+
master_email = resp.organization.master_account_email
|
34
|
+
email = master_email.sub('@', "+#{env}@")
|
35
|
+
end
|
36
|
+
result[:email] = email
|
37
|
+
result[:name] = name
|
38
|
+
|
39
|
+
|
40
|
+
begin
|
41
|
+
logger.info "Creating account named #{name}"
|
42
|
+
resp = org.create_account(account_name: name, email: email)
|
43
|
+
rescue ::Aws::Organizations::Errors::FinalizingOrganizationException
|
44
|
+
logger.info "Waiting to retry account creation as the organization needs to finalize"
|
45
|
+
logger.info "This will eventually succeed after receiving a"
|
46
|
+
logger.info "'Consolidated Billing verification' email from AWS"
|
47
|
+
logger.info "You can leave this running or cancel and restart later."
|
48
|
+
sleep 60
|
49
|
+
retry
|
50
|
+
end
|
51
|
+
|
52
|
+
logger.debug "Created account: #{resp.to_h}"
|
53
|
+
|
54
|
+
status_id = resp.create_account_status.id
|
55
|
+
status = resp.create_account_status.state
|
56
|
+
account_id = resp.create_account_status.account_id
|
57
|
+
|
58
|
+
while status =~ /in_progress/i
|
59
|
+
logger.info "Waiting for account creation to complete, status: #{status}"
|
60
|
+
resp = org.describe_create_account_status(create_account_request_id: status_id)
|
61
|
+
logger.debug("Account creation status check: #{resp.to_h}")
|
62
|
+
status = resp.create_account_status.state
|
63
|
+
account_id = resp.create_account_status.account_id
|
64
|
+
sleep 5
|
65
|
+
end
|
66
|
+
|
67
|
+
if status =~ /failed/i
|
68
|
+
logger.error "Failed to create account: #{resp.create_account_status.failure_reason}"
|
69
|
+
exit(1)
|
70
|
+
end
|
71
|
+
|
72
|
+
result[:account_id] = account_id
|
73
|
+
|
74
|
+
return result
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
@@ -0,0 +1,208 @@
|
|
1
|
+
require_relative '../../../atmos'
|
2
|
+
require_relative '../../../atmos/utils'
|
3
|
+
require_relative '../../../atmos/ui'
|
4
|
+
require_relative '../../../atmos/otp'
|
5
|
+
require 'aws-sdk-core'
|
6
|
+
require 'aws-sdk-iam'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
module Atmos
|
10
|
+
module Providers
|
11
|
+
module Aws
|
12
|
+
|
13
|
+
class AuthManager
|
14
|
+
include GemLogger::LoggerSupport
|
15
|
+
include FileUtils
|
16
|
+
include Atmos::UI
|
17
|
+
|
18
|
+
def initialize(provider)
|
19
|
+
@provider = provider
|
20
|
+
end
|
21
|
+
|
22
|
+
def authenticate(system_env, **opts, &block)
|
23
|
+
|
24
|
+
profile = system_env['AWS_PROFILE']
|
25
|
+
key = system_env['AWS_ACCESS_KEY_ID']
|
26
|
+
secret = system_env['AWS_SECRET_ACCESS_KEY']
|
27
|
+
if profile.blank? && (key.blank? || secret.blank?)
|
28
|
+
logger.warn("An aws profile or key/secret should be supplied via the environment")
|
29
|
+
end
|
30
|
+
|
31
|
+
# Handle bootstrapping a new env account. Newly created organization
|
32
|
+
# accounts only have the default role that can only be assumed by an
|
33
|
+
# iam user, so use that as the target for assume_role, and the root
|
34
|
+
# check below will ensure iam user
|
35
|
+
assume_role_name = nil
|
36
|
+
if opts[:bootstrap] && Atmos.config.atmos_env != 'ops'
|
37
|
+
# TODO: do this hack better
|
38
|
+
assume_role_name = Atmos.config["auth.bootstrap_assume_role_name"]
|
39
|
+
else
|
40
|
+
assume_role_name = opts[:role] || Atmos.config["auth.assume_role_name"]
|
41
|
+
end
|
42
|
+
account_id = Atmos.config.account_hash[Atmos.config.atmos_env].to_s
|
43
|
+
role_arn = "arn:aws:iam::#{account_id}:role/#{assume_role_name}"
|
44
|
+
|
45
|
+
user_name = nil
|
46
|
+
begin
|
47
|
+
sts = ::Aws::STS::Client.new
|
48
|
+
resp = sts.get_caller_identity
|
49
|
+
arn_pieces = resp.arn.split(":")
|
50
|
+
user_name = arn_pieces.last.split("/").last
|
51
|
+
|
52
|
+
# root credentials can't assume role, but they should have full
|
53
|
+
# access for the current account, so proceed (e.g. for bootstrap).
|
54
|
+
if arn_pieces.last == "root"
|
55
|
+
|
56
|
+
# We check the account of the caller to prevent root user of ops
|
57
|
+
# account from bootstrapping an env account, but still allow a
|
58
|
+
# root user of the env account itself to be able to bootstrap
|
59
|
+
# (i.e. to allow not organizational accounts to bootstrap using
|
60
|
+
# their root user)
|
61
|
+
if arn_pieces[-2] != account_id
|
62
|
+
logger.error <<~EOF
|
63
|
+
Account doesn't match credentials. Bootstrapping a new
|
64
|
+
account should be done as an iam user from the ops account or
|
65
|
+
using credentials for a root user of the env account.
|
66
|
+
EOF
|
67
|
+
exit(1)
|
68
|
+
end
|
69
|
+
|
70
|
+
# Should only use root credentials for bootstrap, and thus we
|
71
|
+
# won't have role requirement for mfa, etc, even if root account
|
72
|
+
# uses mfa for login. Thus skip all the other stuff, to
|
73
|
+
# encourage/force use of non-root accounts for normal use
|
74
|
+
logger.warn("Using aws root credentials - should only be neccessary for bootstrap")
|
75
|
+
return block.call(Hash[system_env])
|
76
|
+
end
|
77
|
+
|
78
|
+
rescue ::Aws::STS::Errors::ServiceError => e
|
79
|
+
logger.error "Could not discover aws credentials"
|
80
|
+
exit(1)
|
81
|
+
end
|
82
|
+
|
83
|
+
auth_needed = true
|
84
|
+
cache_key = "#{user_name}-#{assume_role_name}"
|
85
|
+
credentials = read_auth_cache[cache_key]
|
86
|
+
|
87
|
+
if credentials.present?
|
88
|
+
logger.debug("Session cache present, checking expiration...")
|
89
|
+
expiration = Time.parse(credentials['expiration'])
|
90
|
+
session_renew_interval = (session_duration / 4).to_i
|
91
|
+
|
92
|
+
if Time.now > expiration
|
93
|
+
logger.debug "Session cache is expired, performing normal auth"
|
94
|
+
auth_needed = true
|
95
|
+
elsif Time.now > (expiration - session_renew_interval)
|
96
|
+
begin
|
97
|
+
logger.info "Session approaching expiration, renewing..."
|
98
|
+
credentials = assume_role(role_arn, credentials: credentials)
|
99
|
+
auth_needed = false
|
100
|
+
rescue => e
|
101
|
+
logger.info "Failed to renew credentials using session cache, reason: #{e.message}"
|
102
|
+
auth_needed = true
|
103
|
+
end
|
104
|
+
else
|
105
|
+
logger.debug "Session cache is current, skipping auth"
|
106
|
+
auth_needed = false
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
if auth_needed
|
111
|
+
begin
|
112
|
+
logger.info "No active session cache, authenticating..."
|
113
|
+
|
114
|
+
credentials = assume_role(role_arn)
|
115
|
+
write_auth_cache(cache_key => credentials)
|
116
|
+
|
117
|
+
rescue ::Aws::STS::Errors::AccessDenied => e
|
118
|
+
if e.message !~ /explicit deny/
|
119
|
+
logger.debug "Access Denied, reason: #{e.message}"
|
120
|
+
end
|
121
|
+
|
122
|
+
logger.info "Normal auth failed, checking for mfa"
|
123
|
+
|
124
|
+
iam = ::Aws::IAM::Client.new
|
125
|
+
response = iam.list_mfa_devices(user_name: user_name)
|
126
|
+
mfa_serial = response.mfa_devices.first.try(:serial_number)
|
127
|
+
token = nil
|
128
|
+
if mfa_serial.present?
|
129
|
+
|
130
|
+
token = Atmos::Otp.instance.generate(user_name)
|
131
|
+
if token.nil?
|
132
|
+
token = ask("Enter token to retry with mfa: ")
|
133
|
+
else
|
134
|
+
logger.info "Used integrated atmos mfa to generate token"
|
135
|
+
end
|
136
|
+
|
137
|
+
if token.blank?
|
138
|
+
logger.error "A MFA token must be supplied"
|
139
|
+
exit(1)
|
140
|
+
end
|
141
|
+
|
142
|
+
else
|
143
|
+
logger.error "MFA is not setup for your account, retry after doing so"
|
144
|
+
exit(1)
|
145
|
+
end
|
146
|
+
|
147
|
+
credentials = assume_role(role_arn, serial_number: mfa_serial, token_code: token)
|
148
|
+
write_auth_cache(cache_key => credentials)
|
149
|
+
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
process_env = {}
|
154
|
+
process_env['AWS_ACCESS_KEY_ID'] = credentials['access_key_id']
|
155
|
+
process_env['AWS_SECRET_ACCESS_KEY'] = credentials['secret_access_key']
|
156
|
+
process_env['AWS_SESSION_TOKEN'] = credentials['session_token']
|
157
|
+
logger.debug("Calling authentication target with env: #{process_env.inspect}")
|
158
|
+
block.call(Hash[system_env].merge(process_env))
|
159
|
+
end
|
160
|
+
|
161
|
+
private
|
162
|
+
|
163
|
+
def session_duration
|
164
|
+
@session_duration ||= (Atmos.config["auth.session_duration"] || 3600).to_i
|
165
|
+
end
|
166
|
+
|
167
|
+
def assume_role(role_arn, **opts)
|
168
|
+
# use Aws::AssumeRoleCredentials ?
|
169
|
+
if opts[:credentials]
|
170
|
+
c = opts.delete(:credentials)
|
171
|
+
creds = ::Aws::Credentials.new(
|
172
|
+
c[:access_key_id], c[:secret_access_key], c[:session_token]
|
173
|
+
)
|
174
|
+
client_opts = {credentials: creds}
|
175
|
+
else
|
176
|
+
client_opts = {}
|
177
|
+
end
|
178
|
+
sts = ::Aws::STS::Client.new(client_opts)
|
179
|
+
params = {
|
180
|
+
duration_seconds: session_duration,
|
181
|
+
role_session_name: "Atmos",
|
182
|
+
role_arn: role_arn
|
183
|
+
}.merge(opts)
|
184
|
+
logger.debug("Assuming role: #{params}")
|
185
|
+
resp = sts.assume_role(params)
|
186
|
+
return Atmos::Utils::SymbolizedMash.new(resp.credentials.to_h)
|
187
|
+
end
|
188
|
+
|
189
|
+
def auth_cache_file
|
190
|
+
File.join(Atmos.config.auth_cache_dir, 'aws-assume-role.json')
|
191
|
+
end
|
192
|
+
|
193
|
+
def write_auth_cache(h)
|
194
|
+
File.open(auth_cache_file, 'w') do |f|
|
195
|
+
f.puts(JSON.pretty_generate(h))
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def read_auth_cache
|
200
|
+
data = JSON.parse(File.read(auth_cache_file)) rescue {}
|
201
|
+
Atmos::Utils::SymbolizedMash.new(data)
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require_relative '../../../atmos'
|
2
|
+
require 'aws-sdk-ecs'
|
3
|
+
require 'aws-sdk-ecr'
|
4
|
+
require 'open3'
|
5
|
+
|
6
|
+
module Atmos
|
7
|
+
module Providers
|
8
|
+
module Aws
|
9
|
+
|
10
|
+
class ContainerManager
|
11
|
+
include GemLogger::LoggerSupport
|
12
|
+
|
13
|
+
def initialize(provider)
|
14
|
+
@provider = provider
|
15
|
+
end
|
16
|
+
|
17
|
+
def push(ecs_name, local_image,
|
18
|
+
ecr_repo: ecs_name, revision: nil)
|
19
|
+
|
20
|
+
revision = Time.now.strftime('%Y%m%d%H%M%S') unless revision.present?
|
21
|
+
result = {}
|
22
|
+
|
23
|
+
ecr = ::Aws::ECR::Client.new
|
24
|
+
resp = nil
|
25
|
+
|
26
|
+
resp = ecr.get_authorization_token
|
27
|
+
auth_data = resp.authorization_data.first
|
28
|
+
token = auth_data.authorization_token
|
29
|
+
endpoint = auth_data.proxy_endpoint
|
30
|
+
user, password = Base64.decode64(token).split(':')
|
31
|
+
|
32
|
+
# docker login into the ECR repo for the current account so that we can pull/push to it
|
33
|
+
run("docker", "login", "-u", user, "-p", password, endpoint)#, stdin_data: token)
|
34
|
+
|
35
|
+
image="#{ecs_name}:latest"
|
36
|
+
ecs_image="#{endpoint.sub(/https?:\/\//, '')}/#{ecr_repo}"
|
37
|
+
|
38
|
+
tags = ['latest', revision]
|
39
|
+
logger.info "Tagging local image '#{local_image}' with #{tags}"
|
40
|
+
tags.each {|t| run("docker", "tag", local_image, "#{ecs_image}:#{t}") }
|
41
|
+
|
42
|
+
logger.info "Pushing tagged image to ECR repo"
|
43
|
+
tags.each {|t| run("docker", "push", "#{ecs_image}:#{t}") }
|
44
|
+
|
45
|
+
result[:remote_image] = "#{ecs_image}:#{revision}"
|
46
|
+
return result
|
47
|
+
end
|
48
|
+
|
49
|
+
def deploy_task(task, remote_image)
|
50
|
+
result = {}
|
51
|
+
|
52
|
+
ecs = ::Aws::ECS::Client.new
|
53
|
+
resp = nil
|
54
|
+
|
55
|
+
resp = ecs.list_task_definitions(family_prefix: task, sort: 'DESC')
|
56
|
+
latest_defn_arn = resp.task_definition_arns.first
|
57
|
+
|
58
|
+
logger.info "Latest task definition: #{latest_defn_arn}"
|
59
|
+
|
60
|
+
resp = ecs.describe_task_definition(task_definition: latest_defn_arn)
|
61
|
+
latest_defn = resp.task_definition
|
62
|
+
|
63
|
+
new_defn = latest_defn.to_h
|
64
|
+
[:revision, :status, :task_definition_arn,
|
65
|
+
:requires_attributes, :compatibilities].each do |attr|
|
66
|
+
new_defn.delete(attr)
|
67
|
+
end
|
68
|
+
new_defn[:container_definitions].each {|c| c[:image] = remote_image}
|
69
|
+
|
70
|
+
resp = ecs.register_task_definition(**new_defn)
|
71
|
+
result[:task_definition] = resp.task_definition.task_definition_arn
|
72
|
+
|
73
|
+
logger.info "Updated task=#{task} to #{result[:task_definition]} with image #{remote_image}"
|
74
|
+
|
75
|
+
return result
|
76
|
+
end
|
77
|
+
|
78
|
+
def deploy_service(cluster, service, remote_image)
|
79
|
+
result = {}
|
80
|
+
|
81
|
+
ecs = ::Aws::ECS::Client.new
|
82
|
+
resp = nil
|
83
|
+
|
84
|
+
# Get current task definition name from service
|
85
|
+
resp = ecs.describe_services(cluster: cluster, services: [service])
|
86
|
+
current_defn_arn = resp.services.first.task_definition
|
87
|
+
defn_name = current_defn_arn.split("/").last.split(":").first
|
88
|
+
|
89
|
+
logger.info "Current task definition (name=#{defn_name}): #{current_defn_arn}"
|
90
|
+
result = deploy_task(defn_name, remote_image)
|
91
|
+
new_taskdef = result[:task_definition]
|
92
|
+
|
93
|
+
logger.info "Updating service with new task definition: #{new_taskdef}"
|
94
|
+
|
95
|
+
resp = ecs.update_service(cluster: cluster, service: service, task_definition: new_taskdef)
|
96
|
+
|
97
|
+
logger.info "Updated service=#{service} on cluster=#{cluster} to #{new_taskdef} with image #{remote_image}"
|
98
|
+
|
99
|
+
return result
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def run(*args, **opts)
|
105
|
+
logger.debug("Running: #{args}")
|
106
|
+
stdout, status = Open3.capture2e(ENV, *args, **opts)
|
107
|
+
logger.debug(stdout)
|
108
|
+
raise "Failed to run #{args}: #{stdout}" unless status.success?
|
109
|
+
return stdout
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require_relative '../../../atmos'
|
2
|
+
|
3
|
+
Dir.glob(File.join(__dir__, '*.rb')) do |f|
|
4
|
+
require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
|
5
|
+
end
|
6
|
+
|
7
|
+
module Atmos
|
8
|
+
module Providers
|
9
|
+
module Aws
|
10
|
+
|
11
|
+
class Provider
|
12
|
+
include GemLogger::LoggerSupport
|
13
|
+
|
14
|
+
def initialize(name)
|
15
|
+
@name = name
|
16
|
+
end
|
17
|
+
|
18
|
+
def auth_manager
|
19
|
+
@auth_manager ||= begin
|
20
|
+
Atmos::Providers::Aws::AuthManager.new(self)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def user_manager
|
25
|
+
@user_manager ||= begin
|
26
|
+
Atmos::Providers::Aws::UserManager.new(self)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def account_manager
|
31
|
+
@account_manager ||= begin
|
32
|
+
Atmos::Providers::Aws::AccountManager.new(self)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def secret_manager
|
37
|
+
@secret_manager ||= begin
|
38
|
+
Atmos::Providers::Aws::S3SecretManager.new(self)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def container_manager
|
43
|
+
@container_manager ||= begin
|
44
|
+
Atmos::Providers::Aws::ContainerManager.new(self)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|