simplygenius-atmos 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +2 -0
  3. data/LICENSE +13 -0
  4. data/README.md +212 -0
  5. data/exe/atmos +4 -0
  6. data/exe/atmos-docker +12 -0
  7. data/lib/atmos.rb +12 -0
  8. data/lib/atmos/cli.rb +105 -0
  9. data/lib/atmos/commands/account.rb +65 -0
  10. data/lib/atmos/commands/apply.rb +20 -0
  11. data/lib/atmos/commands/auth_exec.rb +29 -0
  12. data/lib/atmos/commands/base_command.rb +12 -0
  13. data/lib/atmos/commands/bootstrap.rb +72 -0
  14. data/lib/atmos/commands/container.rb +58 -0
  15. data/lib/atmos/commands/destroy.rb +18 -0
  16. data/lib/atmos/commands/generate.rb +90 -0
  17. data/lib/atmos/commands/init.rb +18 -0
  18. data/lib/atmos/commands/new.rb +18 -0
  19. data/lib/atmos/commands/otp.rb +54 -0
  20. data/lib/atmos/commands/plan.rb +20 -0
  21. data/lib/atmos/commands/secret.rb +87 -0
  22. data/lib/atmos/commands/terraform.rb +52 -0
  23. data/lib/atmos/commands/user.rb +74 -0
  24. data/lib/atmos/config.rb +208 -0
  25. data/lib/atmos/exceptions.rb +9 -0
  26. data/lib/atmos/generator.rb +199 -0
  27. data/lib/atmos/generator_factory.rb +93 -0
  28. data/lib/atmos/ipc.rb +132 -0
  29. data/lib/atmos/ipc_actions/notify.rb +27 -0
  30. data/lib/atmos/ipc_actions/ping.rb +19 -0
  31. data/lib/atmos/logging.rb +160 -0
  32. data/lib/atmos/otp.rb +61 -0
  33. data/lib/atmos/provider_factory.rb +19 -0
  34. data/lib/atmos/providers/aws/account_manager.rb +82 -0
  35. data/lib/atmos/providers/aws/auth_manager.rb +208 -0
  36. data/lib/atmos/providers/aws/container_manager.rb +116 -0
  37. data/lib/atmos/providers/aws/provider.rb +51 -0
  38. data/lib/atmos/providers/aws/s3_secret_manager.rb +49 -0
  39. data/lib/atmos/providers/aws/user_manager.rb +211 -0
  40. data/lib/atmos/settings_hash.rb +90 -0
  41. data/lib/atmos/terraform_executor.rb +267 -0
  42. data/lib/atmos/ui.rb +159 -0
  43. data/lib/atmos/utils.rb +50 -0
  44. data/lib/atmos/version.rb +3 -0
  45. data/templates/new/config/atmos.yml +50 -0
  46. data/templates/new/config/atmos/runtime.yml +43 -0
  47. data/templates/new/templates.yml +1 -0
  48. metadata +526 -0
@@ -0,0 +1,19 @@
1
+ require_relative '../atmos'
2
+
3
+ module Atmos
4
+ class ProviderFactory
5
+ include GemLogger::LoggerSupport
6
+
7
+ def self.get(name)
8
+ @provider ||= begin
9
+ logger.debug("Loading provider: #{name}")
10
+ require "atmos/providers/#{name}/provider"
11
+ provider = "Atmos::Providers::#{name.camelize}::Provider".constantize
12
+ logger.debug("Loaded provider #{provider}")
13
+ provider.new(name)
14
+ end
15
+ return @provider
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,82 @@
1
+ require_relative '../../../atmos'
2
+ require 'aws-sdk-organizations'
3
+
4
+ module Atmos
5
+ module Providers
6
+ module Aws
7
+
8
+ class AccountManager
9
+ include GemLogger::LoggerSupport
10
+ include FileUtils
11
+
12
+ def initialize(provider)
13
+ @provider = provider
14
+ end
15
+
16
+ def create_account(env, name: nil, email: nil)
17
+ result = {}
18
+ org = ::Aws::Organizations::Client.new
19
+ resp = nil
20
+ name ||= "Atmos #{env} account"
21
+
22
+ begin
23
+ logger.info "Looking up organization"
24
+ resp = org.describe_organization()
25
+ logger.debug "Described organization: #{resp.to_h}"
26
+ rescue ::Aws::Organizations::Errors::AWSOrganizationsNotInUseException
27
+ logger.info "Organization doesn't exist, creating"
28
+ resp = org.create_organization()
29
+ logger.debug "Created organization: #{resp.to_h}"
30
+ end
31
+
32
+ if email.blank?
33
+ master_email = resp.organization.master_account_email
34
+ email = master_email.sub('@', "+#{env}@")
35
+ end
36
+ result[:email] = email
37
+ result[:name] = name
38
+
39
+
40
+ begin
41
+ logger.info "Creating account named #{name}"
42
+ resp = org.create_account(account_name: name, email: email)
43
+ rescue ::Aws::Organizations::Errors::FinalizingOrganizationException
44
+ logger.info "Waiting to retry account creation as the organization needs to finalize"
45
+ logger.info "This will eventually succeed after receiving a"
46
+ logger.info "'Consolidated Billing verification' email from AWS"
47
+ logger.info "You can leave this running or cancel and restart later."
48
+ sleep 60
49
+ retry
50
+ end
51
+
52
+ logger.debug "Created account: #{resp.to_h}"
53
+
54
+ status_id = resp.create_account_status.id
55
+ status = resp.create_account_status.state
56
+ account_id = resp.create_account_status.account_id
57
+
58
+ while status =~ /in_progress/i
59
+ logger.info "Waiting for account creation to complete, status: #{status}"
60
+ resp = org.describe_create_account_status(create_account_request_id: status_id)
61
+ logger.debug("Account creation status check: #{resp.to_h}")
62
+ status = resp.create_account_status.state
63
+ account_id = resp.create_account_status.account_id
64
+ sleep 5
65
+ end
66
+
67
+ if status =~ /failed/i
68
+ logger.error "Failed to create account: #{resp.create_account_status.failure_reason}"
69
+ exit(1)
70
+ end
71
+
72
+ result[:account_id] = account_id
73
+
74
+ return result
75
+ end
76
+
77
+ end
78
+
79
+ end
80
+ end
81
+ end
82
+
@@ -0,0 +1,208 @@
1
+ require_relative '../../../atmos'
2
+ require_relative '../../../atmos/utils'
3
+ require_relative '../../../atmos/ui'
4
+ require_relative '../../../atmos/otp'
5
+ require 'aws-sdk-core'
6
+ require 'aws-sdk-iam'
7
+ require 'json'
8
+
9
+ module Atmos
10
+ module Providers
11
+ module Aws
12
+
13
+ class AuthManager
14
+ include GemLogger::LoggerSupport
15
+ include FileUtils
16
+ include Atmos::UI
17
+
18
+ def initialize(provider)
19
+ @provider = provider
20
+ end
21
+
22
+ def authenticate(system_env, **opts, &block)
23
+
24
+ profile = system_env['AWS_PROFILE']
25
+ key = system_env['AWS_ACCESS_KEY_ID']
26
+ secret = system_env['AWS_SECRET_ACCESS_KEY']
27
+ if profile.blank? && (key.blank? || secret.blank?)
28
+ logger.warn("An aws profile or key/secret should be supplied via the environment")
29
+ end
30
+
31
+ # Handle bootstrapping a new env account. Newly created organization
32
+ # accounts only have the default role that can only be assumed by an
33
+ # iam user, so use that as the target for assume_role, and the root
34
+ # check below will ensure iam user
35
+ assume_role_name = nil
36
+ if opts[:bootstrap] && Atmos.config.atmos_env != 'ops'
37
+ # TODO: do this hack better
38
+ assume_role_name = Atmos.config["auth.bootstrap_assume_role_name"]
39
+ else
40
+ assume_role_name = opts[:role] || Atmos.config["auth.assume_role_name"]
41
+ end
42
+ account_id = Atmos.config.account_hash[Atmos.config.atmos_env].to_s
43
+ role_arn = "arn:aws:iam::#{account_id}:role/#{assume_role_name}"
44
+
45
+ user_name = nil
46
+ begin
47
+ sts = ::Aws::STS::Client.new
48
+ resp = sts.get_caller_identity
49
+ arn_pieces = resp.arn.split(":")
50
+ user_name = arn_pieces.last.split("/").last
51
+
52
+ # root credentials can't assume role, but they should have full
53
+ # access for the current account, so proceed (e.g. for bootstrap).
54
+ if arn_pieces.last == "root"
55
+
56
+ # We check the account of the caller to prevent root user of ops
57
+ # account from bootstrapping an env account, but still allow a
58
+ # root user of the env account itself to be able to bootstrap
59
+ # (i.e. to allow not organizational accounts to bootstrap using
60
+ # their root user)
61
+ if arn_pieces[-2] != account_id
62
+ logger.error <<~EOF
63
+ Account doesn't match credentials. Bootstrapping a new
64
+ account should be done as an iam user from the ops account or
65
+ using credentials for a root user of the env account.
66
+ EOF
67
+ exit(1)
68
+ end
69
+
70
+ # Should only use root credentials for bootstrap, and thus we
71
+ # won't have role requirement for mfa, etc, even if root account
72
+ # uses mfa for login. Thus skip all the other stuff, to
73
+ # encourage/force use of non-root accounts for normal use
74
+ logger.warn("Using aws root credentials - should only be neccessary for bootstrap")
75
+ return block.call(Hash[system_env])
76
+ end
77
+
78
+ rescue ::Aws::STS::Errors::ServiceError => e
79
+ logger.error "Could not discover aws credentials"
80
+ exit(1)
81
+ end
82
+
83
+ auth_needed = true
84
+ cache_key = "#{user_name}-#{assume_role_name}"
85
+ credentials = read_auth_cache[cache_key]
86
+
87
+ if credentials.present?
88
+ logger.debug("Session cache present, checking expiration...")
89
+ expiration = Time.parse(credentials['expiration'])
90
+ session_renew_interval = (session_duration / 4).to_i
91
+
92
+ if Time.now > expiration
93
+ logger.debug "Session cache is expired, performing normal auth"
94
+ auth_needed = true
95
+ elsif Time.now > (expiration - session_renew_interval)
96
+ begin
97
+ logger.info "Session approaching expiration, renewing..."
98
+ credentials = assume_role(role_arn, credentials: credentials)
99
+ auth_needed = false
100
+ rescue => e
101
+ logger.info "Failed to renew credentials using session cache, reason: #{e.message}"
102
+ auth_needed = true
103
+ end
104
+ else
105
+ logger.debug "Session cache is current, skipping auth"
106
+ auth_needed = false
107
+ end
108
+ end
109
+
110
+ if auth_needed
111
+ begin
112
+ logger.info "No active session cache, authenticating..."
113
+
114
+ credentials = assume_role(role_arn)
115
+ write_auth_cache(cache_key => credentials)
116
+
117
+ rescue ::Aws::STS::Errors::AccessDenied => e
118
+ if e.message !~ /explicit deny/
119
+ logger.debug "Access Denied, reason: #{e.message}"
120
+ end
121
+
122
+ logger.info "Normal auth failed, checking for mfa"
123
+
124
+ iam = ::Aws::IAM::Client.new
125
+ response = iam.list_mfa_devices(user_name: user_name)
126
+ mfa_serial = response.mfa_devices.first.try(:serial_number)
127
+ token = nil
128
+ if mfa_serial.present?
129
+
130
+ token = Atmos::Otp.instance.generate(user_name)
131
+ if token.nil?
132
+ token = ask("Enter token to retry with mfa: ")
133
+ else
134
+ logger.info "Used integrated atmos mfa to generate token"
135
+ end
136
+
137
+ if token.blank?
138
+ logger.error "A MFA token must be supplied"
139
+ exit(1)
140
+ end
141
+
142
+ else
143
+ logger.error "MFA is not setup for your account, retry after doing so"
144
+ exit(1)
145
+ end
146
+
147
+ credentials = assume_role(role_arn, serial_number: mfa_serial, token_code: token)
148
+ write_auth_cache(cache_key => credentials)
149
+
150
+ end
151
+ end
152
+
153
+ process_env = {}
154
+ process_env['AWS_ACCESS_KEY_ID'] = credentials['access_key_id']
155
+ process_env['AWS_SECRET_ACCESS_KEY'] = credentials['secret_access_key']
156
+ process_env['AWS_SESSION_TOKEN'] = credentials['session_token']
157
+ logger.debug("Calling authentication target with env: #{process_env.inspect}")
158
+ block.call(Hash[system_env].merge(process_env))
159
+ end
160
+
161
+ private
162
+
163
+ def session_duration
164
+ @session_duration ||= (Atmos.config["auth.session_duration"] || 3600).to_i
165
+ end
166
+
167
+ def assume_role(role_arn, **opts)
168
+ # use Aws::AssumeRoleCredentials ?
169
+ if opts[:credentials]
170
+ c = opts.delete(:credentials)
171
+ creds = ::Aws::Credentials.new(
172
+ c[:access_key_id], c[:secret_access_key], c[:session_token]
173
+ )
174
+ client_opts = {credentials: creds}
175
+ else
176
+ client_opts = {}
177
+ end
178
+ sts = ::Aws::STS::Client.new(client_opts)
179
+ params = {
180
+ duration_seconds: session_duration,
181
+ role_session_name: "Atmos",
182
+ role_arn: role_arn
183
+ }.merge(opts)
184
+ logger.debug("Assuming role: #{params}")
185
+ resp = sts.assume_role(params)
186
+ return Atmos::Utils::SymbolizedMash.new(resp.credentials.to_h)
187
+ end
188
+
189
+ def auth_cache_file
190
+ File.join(Atmos.config.auth_cache_dir, 'aws-assume-role.json')
191
+ end
192
+
193
+ def write_auth_cache(h)
194
+ File.open(auth_cache_file, 'w') do |f|
195
+ f.puts(JSON.pretty_generate(h))
196
+ end
197
+ end
198
+
199
+ def read_auth_cache
200
+ data = JSON.parse(File.read(auth_cache_file)) rescue {}
201
+ Atmos::Utils::SymbolizedMash.new(data)
202
+ end
203
+
204
+ end
205
+
206
+ end
207
+ end
208
+ end
@@ -0,0 +1,116 @@
1
+ require_relative '../../../atmos'
2
+ require 'aws-sdk-ecs'
3
+ require 'aws-sdk-ecr'
4
+ require 'open3'
5
+
6
+ module Atmos
7
+ module Providers
8
+ module Aws
9
+
10
+ class ContainerManager
11
+ include GemLogger::LoggerSupport
12
+
13
+ def initialize(provider)
14
+ @provider = provider
15
+ end
16
+
17
+ def push(ecs_name, local_image,
18
+ ecr_repo: ecs_name, revision: nil)
19
+
20
+ revision = Time.now.strftime('%Y%m%d%H%M%S') unless revision.present?
21
+ result = {}
22
+
23
+ ecr = ::Aws::ECR::Client.new
24
+ resp = nil
25
+
26
+ resp = ecr.get_authorization_token
27
+ auth_data = resp.authorization_data.first
28
+ token = auth_data.authorization_token
29
+ endpoint = auth_data.proxy_endpoint
30
+ user, password = Base64.decode64(token).split(':')
31
+
32
+ # docker login into the ECR repo for the current account so that we can pull/push to it
33
+ run("docker", "login", "-u", user, "-p", password, endpoint)#, stdin_data: token)
34
+
35
+ image="#{ecs_name}:latest"
36
+ ecs_image="#{endpoint.sub(/https?:\/\//, '')}/#{ecr_repo}"
37
+
38
+ tags = ['latest', revision]
39
+ logger.info "Tagging local image '#{local_image}' with #{tags}"
40
+ tags.each {|t| run("docker", "tag", local_image, "#{ecs_image}:#{t}") }
41
+
42
+ logger.info "Pushing tagged image to ECR repo"
43
+ tags.each {|t| run("docker", "push", "#{ecs_image}:#{t}") }
44
+
45
+ result[:remote_image] = "#{ecs_image}:#{revision}"
46
+ return result
47
+ end
48
+
49
+ def deploy_task(task, remote_image)
50
+ result = {}
51
+
52
+ ecs = ::Aws::ECS::Client.new
53
+ resp = nil
54
+
55
+ resp = ecs.list_task_definitions(family_prefix: task, sort: 'DESC')
56
+ latest_defn_arn = resp.task_definition_arns.first
57
+
58
+ logger.info "Latest task definition: #{latest_defn_arn}"
59
+
60
+ resp = ecs.describe_task_definition(task_definition: latest_defn_arn)
61
+ latest_defn = resp.task_definition
62
+
63
+ new_defn = latest_defn.to_h
64
+ [:revision, :status, :task_definition_arn,
65
+ :requires_attributes, :compatibilities].each do |attr|
66
+ new_defn.delete(attr)
67
+ end
68
+ new_defn[:container_definitions].each {|c| c[:image] = remote_image}
69
+
70
+ resp = ecs.register_task_definition(**new_defn)
71
+ result[:task_definition] = resp.task_definition.task_definition_arn
72
+
73
+ logger.info "Updated task=#{task} to #{result[:task_definition]} with image #{remote_image}"
74
+
75
+ return result
76
+ end
77
+
78
+ def deploy_service(cluster, service, remote_image)
79
+ result = {}
80
+
81
+ ecs = ::Aws::ECS::Client.new
82
+ resp = nil
83
+
84
+ # Get current task definition name from service
85
+ resp = ecs.describe_services(cluster: cluster, services: [service])
86
+ current_defn_arn = resp.services.first.task_definition
87
+ defn_name = current_defn_arn.split("/").last.split(":").first
88
+
89
+ logger.info "Current task definition (name=#{defn_name}): #{current_defn_arn}"
90
+ result = deploy_task(defn_name, remote_image)
91
+ new_taskdef = result[:task_definition]
92
+
93
+ logger.info "Updating service with new task definition: #{new_taskdef}"
94
+
95
+ resp = ecs.update_service(cluster: cluster, service: service, task_definition: new_taskdef)
96
+
97
+ logger.info "Updated service=#{service} on cluster=#{cluster} to #{new_taskdef} with image #{remote_image}"
98
+
99
+ return result
100
+ end
101
+
102
+ private
103
+
104
+ def run(*args, **opts)
105
+ logger.debug("Running: #{args}")
106
+ stdout, status = Open3.capture2e(ENV, *args, **opts)
107
+ logger.debug(stdout)
108
+ raise "Failed to run #{args}: #{stdout}" unless status.success?
109
+ return stdout
110
+ end
111
+
112
+ end
113
+
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,51 @@
1
+ require_relative '../../../atmos'
2
+
3
+ Dir.glob(File.join(__dir__, '*.rb')) do |f|
4
+ require_relative "#{File.basename(f).sub(/\.rb$/, "")}"
5
+ end
6
+
7
+ module Atmos
8
+ module Providers
9
+ module Aws
10
+
11
+ class Provider
12
+ include GemLogger::LoggerSupport
13
+
14
+ def initialize(name)
15
+ @name = name
16
+ end
17
+
18
+ def auth_manager
19
+ @auth_manager ||= begin
20
+ Atmos::Providers::Aws::AuthManager.new(self)
21
+ end
22
+ end
23
+
24
+ def user_manager
25
+ @user_manager ||= begin
26
+ Atmos::Providers::Aws::UserManager.new(self)
27
+ end
28
+ end
29
+
30
+ def account_manager
31
+ @account_manager ||= begin
32
+ Atmos::Providers::Aws::AccountManager.new(self)
33
+ end
34
+ end
35
+
36
+ def secret_manager
37
+ @secret_manager ||= begin
38
+ Atmos::Providers::Aws::S3SecretManager.new(self)
39
+ end
40
+ end
41
+
42
+ def container_manager
43
+ @container_manager ||= begin
44
+ Atmos::Providers::Aws::ContainerManager.new(self)
45
+ end
46
+ end
47
+ end
48
+
49
+ end
50
+ end
51
+ end