elastic_beans 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.rspec +3 -0
  4. data/Gemfile +9 -0
  5. data/LICENSE.md +21 -0
  6. data/README.md +184 -0
  7. data/Rakefile +6 -0
  8. data/bin/console +14 -0
  9. data/bin/setup +8 -0
  10. data/circle.yml +20 -0
  11. data/elastic_beans.gemspec +31 -0
  12. data/exe/beans +198 -0
  13. data/lib/elastic_beans/application.rb +127 -0
  14. data/lib/elastic_beans/application_version.rb +202 -0
  15. data/lib/elastic_beans/aws/cloudformation_stack.rb +66 -0
  16. data/lib/elastic_beans/command/configure.rb +184 -0
  17. data/lib/elastic_beans/command/create.rb +150 -0
  18. data/lib/elastic_beans/command/deploy.rb +77 -0
  19. data/lib/elastic_beans/command/exec.rb +37 -0
  20. data/lib/elastic_beans/command/set_env.rb +77 -0
  21. data/lib/elastic_beans/command/talk.rb +74 -0
  22. data/lib/elastic_beans/command/version.rb +17 -0
  23. data/lib/elastic_beans/command.rb +12 -0
  24. data/lib/elastic_beans/configuration_template/base.rb +114 -0
  25. data/lib/elastic_beans/configuration_template/exec.rb +50 -0
  26. data/lib/elastic_beans/configuration_template/scheduler.rb +20 -0
  27. data/lib/elastic_beans/configuration_template/webserver.rb +49 -0
  28. data/lib/elastic_beans/configuration_template/worker.rb +27 -0
  29. data/lib/elastic_beans/configuration_template.rb +197 -0
  30. data/lib/elastic_beans/dns_entry.rb +127 -0
  31. data/lib/elastic_beans/environment/exec.rb +23 -0
  32. data/lib/elastic_beans/environment/scheduler.rb +23 -0
  33. data/lib/elastic_beans/environment/webserver.rb +23 -0
  34. data/lib/elastic_beans/environment/worker.rb +29 -0
  35. data/lib/elastic_beans/environment.rb +300 -0
  36. data/lib/elastic_beans/error/environments_not_ready.rb +15 -0
  37. data/lib/elastic_beans/error.rb +15 -0
  38. data/lib/elastic_beans/exec/ebextension.yml +10 -0
  39. data/lib/elastic_beans/exec/elastic_beans_exec.conf +15 -0
  40. data/lib/elastic_beans/exec/init.rb +50 -0
  41. data/lib/elastic_beans/exec/run_command.sh +13 -0
  42. data/lib/elastic_beans/exec/sqs_consumer.rb +75 -0
  43. data/lib/elastic_beans/network.rb +44 -0
  44. data/lib/elastic_beans/rack/exec.rb +63 -0
  45. data/lib/elastic_beans/scheduler/ebextension.yml +4 -0
  46. data/lib/elastic_beans/ui.rb +31 -0
  47. data/lib/elastic_beans/version.rb +3 -0
  48. data/lib/elastic_beans.rb +9 -0
  49. metadata +218 -0
@@ -0,0 +1,127 @@
1
+ require "json"
2
+ require "aws-sdk"
3
+ require "elastic_beans/aws/cloudformation_stack"
4
+ require "elastic_beans/error"
5
+
6
+ module ElasticBeans
7
+ class Application
8
+ attr_reader :name
9
+
10
+ def initialize(name:, cloudformation:, elastic_beanstalk:)
11
+ @name = name
12
+ @elastic_beanstalk = elastic_beanstalk
13
+ @stack = ElasticBeans::Aws::CloudformationStack.new(name, cloudformation: cloudformation)
14
+ end
15
+
16
+ def deployed_version
17
+ response = elastic_beanstalk.describe_environments(application_name: name)
18
+ live_environments = response.environments.select { |environment| environment.status !~ /Terminat/ }
19
+ environment = live_environments.max_by(&:date_updated)
20
+ if environment
21
+ ElasticBeans::ApplicationVersion.new(environment.version_label)
22
+ end
23
+ rescue ::Aws::ElasticBeanstalk::Errors::Throttling
24
+ sleep 5
25
+ retry
26
+ end
27
+
28
+ def configuration_templates
29
+ response = elastic_beanstalk.describe_applications(application_names: [name])
30
+ application = response.applications[0]
31
+ unless application
32
+ raise MissingApplicationError
33
+ end
34
+
35
+ templates = application.configuration_templates
36
+ templates.map { |template_name|
37
+ ElasticBeans::ConfigurationTemplate.new_from_existing(
38
+ template_name,
39
+ application: self,
40
+ elastic_beanstalk: elastic_beanstalk,
41
+ )
42
+ }
43
+ rescue ::Aws::ElasticBeanstalk::Errors::Throttling
44
+ sleep 5
45
+ retry
46
+ end
47
+
48
+ def environments
49
+ response = elastic_beanstalk.describe_environments(application_name: name)
50
+ live_environments = response.environments.select { |environment| environment.status !~ /Terminat/ }
51
+ live_environments.map { |environment|
52
+ ElasticBeans::Environment.new_from_existing(
53
+ environment,
54
+ application: self,
55
+ elastic_beanstalk: elastic_beanstalk,
56
+ )
57
+ }
58
+ rescue ::Aws::ElasticBeanstalk::Errors::Throttling
59
+ sleep 5
60
+ retry
61
+ end
62
+
63
+ def enqueue_command(command, sqs:)
64
+ if environments.none? { |environment| environment.is_a?(Environment::Exec) }
65
+ raise MissingExecEnvironmentError
66
+ end
67
+
68
+ sqs.send_message(
69
+ queue_url: exec_queue_url,
70
+ message_body: exec_message(command),
71
+ )
72
+ end
73
+
74
+ def exec_queue_url
75
+ stack.stack_output("ExecQueueUrl")
76
+ end
77
+
78
+ def versions
79
+ response = elastic_beanstalk.describe_application_versions(application_name: name)
80
+ response.application_versions.map { |version| ElasticBeans::ApplicationVersion.new(version.version_label) }
81
+ rescue ::Aws::ElasticBeanstalk::Errors::Throttling
82
+ sleep 5
83
+ retry
84
+ end
85
+
86
+ def worker_queue_url(queue)
87
+ queue_attribute = queue.to_s.downcase
88
+ queue_attribute[0] = queue_attribute[0].upcase
89
+ stack.stack_output("Worker#{queue_attribute}QueueUrl")
90
+ end
91
+
92
+ def worker_queues
93
+ stack.stack_outputs.each_with_object([]) do |(output_key, _), acc|
94
+ match = /\AWorker(?<queue>\w+)QueueUrl\z/.match(output_key)
95
+ if match
96
+ acc << match[:queue].downcase
97
+ end
98
+ end
99
+ end
100
+
101
+ private
102
+
103
+ attr_reader :elastic_beanstalk, :stack
104
+
105
+ def exec_message(command)
106
+ {
107
+ command: command,
108
+ }.to_json
109
+ end
110
+
111
+ class MissingApplicationError < ElasticBeans::Error
112
+ def message
113
+ "Application `#{@application_name}' does not exist. Please create the Elastic Beanstalk application using a CloudFormation stack."
114
+ end
115
+ end
116
+
117
+ class MissingExecEnvironmentError < ElasticBeans::Error
118
+ def message
119
+ <<-MESSAGE
120
+ A one-off command cannot be executed because the "exec" environment does not exist. Please create it:
121
+
122
+ #{$0} create -a APPLICATION exec
123
+ MESSAGE
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,202 @@
1
+ require "fileutils"
2
+ require "timeout"
3
+ require "aws-sdk"
4
+ require "zip"
5
+ require "nesty"
6
+ require "elastic_beans/error"
7
+
8
+ module ElasticBeans
9
+ class ApplicationVersion
10
+ attr_reader :version_label
11
+
12
+ def initialize(version_label)
13
+ @version_label = version_label
14
+ end
15
+
16
+ class << self
17
+ def create(working_directory:, application:, elastic_beanstalk:, s3:)
18
+ version_label = fetch_version_label(working_directory)
19
+ if application.versions.any? { |version| version.version_label == version_label }
20
+ return new(version_label)
21
+ end
22
+
23
+ archive_path = File.join(working_directory, ".elasticbeanstalk", "#{version_label}.zip")
24
+ create_archive(sha: version_label, path: archive_path)
25
+ create_application_version(
26
+ path: archive_path,
27
+ version_label: version_label,
28
+ application: application,
29
+ elastic_beanstalk: elastic_beanstalk,
30
+ s3: s3,
31
+ )
32
+ new(version_label)
33
+ end
34
+
35
+ private
36
+
37
+ def create_application_version(path:, version_label:, application:, elastic_beanstalk:, s3:)
38
+ archive_filename = File.basename(path)
39
+ archive = File.new(path)
40
+ eb_bucket_name = find_eb_bucket_name(s3)
41
+ s3.put_object(body: archive, bucket: eb_bucket_name, key: archive_filename)
42
+ elastic_beanstalk.create_application_version(
43
+ application_name: application.name,
44
+ source_bundle: {s3_bucket: eb_bucket_name, s3_key: archive_filename},
45
+ version_label: version_label,
46
+ process: true,
47
+ )
48
+
49
+ status = "PROCESSING"
50
+ Timeout.timeout(600) do
51
+ while status == "PROCESSING"
52
+ sleep 5
53
+ response = elastic_beanstalk.describe_application_versions(
54
+ application_name: application.name,
55
+ version_labels: [version_label],
56
+ )
57
+ status = response.application_versions[0].status
58
+ end
59
+ end
60
+ if status != "PROCESSED"
61
+ raise FailedApplicationVersion.new(version_label: version_label, status: status)
62
+ end
63
+ rescue ::Aws::ElasticBeanstalk::Errors::Throttling
64
+ sleep 5
65
+ retry
66
+ rescue ::Aws::Errors::ServiceError => e
67
+ raise FailedApplicationVersion.new(version_label: version_label, cause: e)
68
+ rescue Timeout::Error
69
+ raise FailedApplicationVersion.new(version_label: version_label, status: status)
70
+ end
71
+
72
+ def create_archive(sha:, path:)
73
+ FileUtils.mkdir_p(File.dirname(path))
74
+ system("git archive --format=zip -o #{path} #{sha}")
75
+ unless $?.success?
76
+ raise FailedArchiveError.new(sha: sha, path: path)
77
+ end
78
+
79
+ Zip::File.open(path) do |zip_file|
80
+ inject_exec_into_zip(zip_file)
81
+ inject_scheduler_into_zip(zip_file)
82
+ end
83
+ end
84
+
85
+ def find_eb_bucket_name(s3)
86
+ bucket = s3.list_buckets.buckets.find { |bucket| bucket.name.start_with?("elasticbeanstalk-") }
87
+ unless bucket
88
+ raise MissingBucketError
89
+ end
90
+ bucket.name
91
+ end
92
+
93
+ def fetch_version_label(working_directory)
94
+ if Dir.glob(File.join(working_directory, ".git")).empty?
95
+ raise InvalidVersionWorkingDirectoryError.new(wd: working_directory)
96
+ end
97
+
98
+ Dir.chdir(working_directory) do
99
+ `git rev-parse HEAD`.chomp
100
+ end
101
+ end
102
+
103
+ def inject_exec_into_zip(zip_file)
104
+ begin
105
+ zip_file.mkdir(".elastic_beans")
106
+ rescue Errno::EEXIST
107
+ end
108
+ begin
109
+ zip_file.mkdir(".elastic_beans/exec")
110
+ rescue Errno::EEXIST
111
+ end
112
+ zip_file.add(
113
+ ".elastic_beans/exec/elastic_beans_exec.conf",
114
+ File.expand_path('../exec/elastic_beans_exec.conf', __FILE__),
115
+ )
116
+ zip_file.add(
117
+ ".elastic_beans/exec/init.rb",
118
+ File.expand_path('../exec/init.rb', __FILE__),
119
+ )
120
+ zip_file.add(
121
+ ".elastic_beans/exec/run_command.sh",
122
+ File.expand_path('../exec/run_command.sh', __FILE__),
123
+ )
124
+ zip_file.add(
125
+ ".elastic_beans/exec/sqs_consumer.rb",
126
+ File.expand_path('../exec/sqs_consumer.rb', __FILE__),
127
+ )
128
+
129
+ begin
130
+ zip_file.mkdir(".ebextensions")
131
+ rescue Errno::EEXIST
132
+ end
133
+ zip_file.add(
134
+ ".ebextensions/elastic_beans_exec.config",
135
+ File.expand_path('../exec/ebextension.yml', __FILE__),
136
+ )
137
+ end
138
+
139
+ def inject_scheduler_into_zip(zip_file)
140
+ begin
141
+ zip_file.mkdir(".ebextensions")
142
+ rescue Errno::EEXIST
143
+ end
144
+ zip_file.add(
145
+ ".ebextensions/elastic_beans_scheduler.config",
146
+ File.expand_path('../scheduler/ebextension.yml', __FILE__),
147
+ )
148
+ end
149
+ end
150
+
151
+ def ==(other)
152
+ version_label == other.version_label
153
+ end
154
+
155
+ private
156
+
157
+ class FailedApplicationVersion < ElasticBeans::Error
158
+ include Nesty::NestedError
159
+
160
+ def initialize(version_label:, status: nil, cause: nil)
161
+ @version_label = version_label
162
+ @status = status
163
+ @nested = cause
164
+ end
165
+
166
+ def message
167
+ msg = "Application version `#{@version_label}' failed"
168
+ msg << "\nVersion status is `#{@status}'" if @status
169
+ msg << "\nThe error from AWS was \"#{@nested.message}\"" if @nested
170
+ msg
171
+ end
172
+ end
173
+
174
+ class FailedArchiveError < ElasticBeans::Error
175
+ def initialize(sha:, path:)
176
+ @sha = sha
177
+ @path = path
178
+ end
179
+
180
+ def message
181
+ "Could not create git archive of sha `#{sha}' at path `#{path}'"
182
+ end
183
+ end
184
+
185
+ class InvalidVersionWorkingDirectoryError < ElasticBeans::Error
186
+ def initialize(wd:)
187
+ @wd = wd
188
+ end
189
+
190
+ def message
191
+ "Cannot create a version from '#{@wd}`, please change to a Rails project root directory."
192
+ end
193
+ end
194
+
195
+ class MissingBucketError < ElasticBeans::Error
196
+ def message
197
+ "Cannot find the Elastic Beanstalk S3 bucket." \
198
+ " Create an S3 bucket with a name starting with \"elasticbeanstalk-\"."
199
+ end
200
+ end
201
+ end
202
+ end
@@ -0,0 +1,66 @@
1
+ require "aws-sdk"
2
+ require "elastic_beans/error"
3
+
4
+ module ElasticBeans
5
+ module Aws
6
+ class CloudformationStack
7
+ STACK_STATUSES = %w(CREATE_COMPLETE UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE)
8
+
9
+ def initialize(stack_name, cloudformation:)
10
+ @stack_name = stack_name
11
+ @cloudformation = cloudformation
12
+ end
13
+
14
+ def stack_output(output_key)
15
+ output = stack.outputs.find { |o| o.output_key == output_key }
16
+ unless output
17
+ raise MissingOutputError.new(stack_name: stack_name, output_key: output_key)
18
+ end
19
+ output.output_value
20
+ end
21
+
22
+ def stack_outputs
23
+ stack.outputs.reduce({}) { |a, e| a.merge(e.output_key => e.output_value) }
24
+ end
25
+
26
+ private
27
+
28
+ attr_reader :stack_name, :cloudformation
29
+
30
+ def stack
31
+ s = cloudformation.describe_stacks(stack_name: stack_name).stacks.find { |stack|
32
+ STACK_STATUSES.include?(stack.stack_status)
33
+ }
34
+ unless s
35
+ raise MissingStackError.new(stack_name: stack_name)
36
+ end
37
+ s
38
+ rescue ::Aws::CloudFormation::Errors::ValidationError
39
+ raise MissingStackError.new(stack_name: stack_name)
40
+ end
41
+
42
+ class MissingStackError < ElasticBeans::Error
43
+ def initialize(stack_name:)
44
+ @stack_name = stack_name
45
+ end
46
+
47
+ def message
48
+ "CloudFormation stack `#{@stack_name}' not found." \
49
+ " Make sure the stack exists and matches the outputs required."
50
+ end
51
+ end
52
+
53
+ class MissingOutputError < ElasticBeans::Error
54
+ def initialize(stack_name:, output_key:)
55
+ @stack_name = stack_name
56
+ @output_key = output_key
57
+ end
58
+
59
+ def message
60
+ "Stack `#{@stack_name}' is missing output `#{@output_key}'." \
61
+ " Make sure the stack matches the outputs required."
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,184 @@
1
+ require "elastic_beans/error/environments_not_ready"
2
+
3
+ module ElasticBeans
4
+ module Command
5
+ class Configure
6
+ USAGE = "configure -n NETWORK -a APPLICATION [-k KEYPAIR] [-p PUBLIC_KEY] [-s SSL_CERTIFICATE_ID] [-i IMAGE_ID] [-t INSTANCE_TYPE] [-l EXEC_LOGGING_HTTP_ENDPOINT]"
7
+ DESC = "Configure the given Elastic Beanstalk application"
8
+ LONG_DESC = <<-LONG_DESC
9
+ Configure the given Elastic Beanstalk application.
10
+ Creates and updates configuration templates for each possible environment type.
11
+ Updates running environments with the new configuration.
12
+
13
+ Requires the networking CloudFormation stack and application name.
14
+ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
15
+ Some environment variables that Rails always requires must be set:
16
+
17
+ * DATABASE_URL
18
+ * SECRET_KEY_BASE
19
+ LONG_DESC
20
+
21
+ def initialize(
22
+ image_id:,
23
+ instance_type:,
24
+ keypair:,
25
+ logging_endpoint:,
26
+ public_key:,
27
+ ssl_certificate_id:,
28
+ application:,
29
+ network:,
30
+ elastic_beanstalk:,
31
+ iam:,
32
+ ui:
33
+ )
34
+ @image_id = image_id
35
+ @instance_type = instance_type
36
+ @keypair = keypair
37
+ @logging_endpoint = logging_endpoint
38
+ @public_key = public_key
39
+ @ssl_certificate_id = ssl_certificate_id
40
+ @application = application
41
+ @network = network
42
+ @elastic_beanstalk = elastic_beanstalk
43
+ @iam = iam
44
+ @ui = ui
45
+ end
46
+
47
+ def run
48
+ environments = application.environments
49
+ unready_environments = environments.select { |environment| environment.status != "Ready" }
50
+ if unready_environments.any?
51
+ raise EnvironmentsNotReady.new(environments: unready_environments)
52
+ end
53
+
54
+ progressbar = ProgressBar.create(title: "Configuring", total: nil, output: ui.stdout)
55
+ progressbar.log("Updating configuration templates in #{application.name}...")
56
+ progressbar.log("Updating base configuration template...")
57
+ base_config = ElasticBeans::ConfigurationTemplate::Base.new(
58
+ application: application,
59
+ elastic_beanstalk: elastic_beanstalk,
60
+ )
61
+ base_config.upsert(
62
+ network: network,
63
+ database_url: database_url,
64
+ secret_key_base: secret_key_base,
65
+ image_id: image_id,
66
+ instance_type: instance_type,
67
+ keypair: keypair,
68
+ iam: iam,
69
+ )
70
+ progressbar.increment
71
+ progressbar.log("Updating webserver configuration template...")
72
+ webserver_config = ElasticBeans::ConfigurationTemplate::Webserver.new(
73
+ application: application,
74
+ elastic_beanstalk: elastic_beanstalk,
75
+ )
76
+ webserver_config.upsert(
77
+ network: network,
78
+ database_url: database_url,
79
+ secret_key_base: secret_key_base,
80
+ image_id: image_id,
81
+ instance_type: instance_type,
82
+ keypair: keypair,
83
+ iam: iam,
84
+ public_key: public_key,
85
+ ssl_certificate_id: ssl_certificate_id,
86
+ )
87
+ progressbar.increment
88
+ progressbar.log("Updating exec configuration template...")
89
+ exec_config = ElasticBeans::ConfigurationTemplate::Exec.new(
90
+ application: application,
91
+ elastic_beanstalk: elastic_beanstalk,
92
+ )
93
+ exec_config.upsert(
94
+ network: network,
95
+ database_url: database_url,
96
+ secret_key_base: secret_key_base,
97
+ image_id: image_id,
98
+ instance_type: instance_type,
99
+ keypair: keypair,
100
+ iam: iam,
101
+ logging_endpoint: logging_endpoint,
102
+ )
103
+ progressbar.increment
104
+ progressbar.log("Updating scheduler configuration template...")
105
+ scheduler_config = ElasticBeans::ConfigurationTemplate::Scheduler.new(
106
+ application: application,
107
+ elastic_beanstalk: elastic_beanstalk,
108
+ )
109
+ scheduler_config.upsert(
110
+ network: network,
111
+ database_url: database_url,
112
+ secret_key_base: secret_key_base,
113
+ image_id: image_id,
114
+ instance_type: instance_type,
115
+ keypair: keypair,
116
+ iam: iam,
117
+ )
118
+ progressbar.increment
119
+ application.worker_queues.each do |queue|
120
+ progressbar.log("Updating worker-#{queue} configuration template...")
121
+ worker_config = ElasticBeans::ConfigurationTemplate::Worker.new(
122
+ queue: queue,
123
+ application: application,
124
+ elastic_beanstalk: elastic_beanstalk,
125
+ )
126
+ worker_config.upsert(
127
+ network: network,
128
+ database_url: database_url,
129
+ secret_key_base: secret_key_base,
130
+ image_id: image_id,
131
+ instance_type: instance_type,
132
+ keypair: keypair,
133
+ iam: iam,
134
+ )
135
+ progressbar.increment
136
+ end
137
+
138
+ threads = environments.map { |environment|
139
+ progressbar.log("Updating `#{environment.name}'...")
140
+ thread = Thread.new do
141
+ environment.update_configuration
142
+ end
143
+ progressbar.increment
144
+ thread
145
+ }
146
+
147
+ loop do
148
+ sleep 1
149
+ progressbar.increment
150
+ if threads.none?(&:alive?)
151
+ progressbar.total = progressbar.progress
152
+ break
153
+ end
154
+ end
155
+
156
+ threads.each(&:join)
157
+ end
158
+
159
+ private
160
+
161
+ attr_reader(
162
+ :application,
163
+ :image_id,
164
+ :instance_type,
165
+ :keypair,
166
+ :logging_endpoint,
167
+ :network,
168
+ :public_key,
169
+ :ssl_certificate_id,
170
+ :elastic_beanstalk,
171
+ :iam,
172
+ :ui,
173
+ )
174
+
175
+ def database_url
176
+ ENV['DATABASE_URL']
177
+ end
178
+
179
+ def secret_key_base
180
+ ENV['SECRET_KEY_BASE']
181
+ end
182
+ end
183
+ end
184
+ end