qops 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +103 -0
- data/bin/qops +10 -0
- data/lib/qops.rb +32 -0
- data/lib/qops/cookbook/cookbook.rb +160 -0
- data/lib/qops/deployment/app.rb +117 -0
- data/lib/qops/deployment/helpers.rb +74 -0
- data/lib/qops/deployment/instances.rb +351 -0
- data/lib/qops/environment.rb +110 -0
- data/lib/qops/helpers.rb +153 -0
- data/lib/qops/version.rb +4 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5a359bd27c24df1c4c124864fdc9d409bf43e453
|
4
|
+
data.tar.gz: 3d42a3143c38fde591eae2969d8abe4d0be36b20
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4ca05c0d395bbcadfc399ad7a52649f19ebf68520c09acd37bb56c0d86bb9c77cfe0c65dd7639a3c1e6b43c2f9b6f460158a9d94374129806f6832b03b1df78d
|
7
|
+
data.tar.gz: cfb7726f54f320898ba5c955babb1d1a17bdbcc5695612a740dbb347f743bd49a82fe51264b59184bd638d89484b73423ea7e101372752a821b3deed301eb93c
|
data/README.md
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
# Qops : Quandl Operations Helper
|
2
|
+
|
3
|
+
## Configuring Qops in your project
|
4
|
+
|
5
|
+
1. In your repo's config directory create a file called opsworks.yml. See sample.
|
6
|
+
2. Install gem `qops` to your system. Be sure to use gem install `qops`. Do not include it as part of the bundle of your repo as it is mean't to be run outside of the scope of your project.
|
7
|
+
3. Run `qops list` to get a list of commands you can run.
|
8
|
+
4. Run `qops help <command>` for more information on command argument options
|
9
|
+
|
10
|
+
## FAQ:
|
11
|
+
|
12
|
+
### Q: The `qops` gem is currently not public. How do I access it?
|
13
|
+
|
14
|
+
Please add your personal gemfury source to the gem path to install it.
|
15
|
+
|
16
|
+
### Q: For the `qops qops:instance:run_command` command, it provides two options: one is run commands against all instances of the stack all in once, one is run commands on each instances of the stack one by one randomly. How do I use this?
|
17
|
+
|
18
|
+
When running commands one by one, between each execution of the command, there will be a delay. The delay is config by wait_deploy. By default it is 180 seconds when it is not defined. For now, run_command command will only support commends `setup` `configure` `install_dependencies` `update_dependencies`, since commands `update_custom_cookbooks` `deploy` was implemented before.
|
19
|
+
|
20
|
+
### Q: How do I use QOPS to override env variables on my opsworks CHEF 11 stack?
|
21
|
+
|
22
|
+
You can use the custom json flag for this. Example:
|
23
|
+
|
24
|
+
```qops qops:instance:up -e staging -j '{ "deploy" : { "wikiposit" : { "environment_variables" : { "ENV_VARIABLE_TO_OVERIDE": "X" } } } }'```
|
25
|
+
|
26
|
+
In this case the `wikiposit` is the opsworks app wikiposit. You will need to change this to whichever app you are deploying.
|
27
|
+
|
28
|
+
See also: http://docs.aws.amazon.com/opsworks/latest/userguide/apps-environment-vars.html
|
29
|
+
|
30
|
+
## Sample Config (with all options)
|
31
|
+
|
32
|
+
```
|
33
|
+
_daily_schedule: &daily_schedule
|
34
|
+
'13': 'on'
|
35
|
+
'14': 'on'
|
36
|
+
'15': 'on'
|
37
|
+
'16': 'on'
|
38
|
+
'17': 'on'
|
39
|
+
'18': 'on'
|
40
|
+
'19': 'on'
|
41
|
+
'20': 'on'
|
42
|
+
'21': 'on'
|
43
|
+
'22': 'on'
|
44
|
+
|
45
|
+
_weekly_schedule: &weekly_schedule
|
46
|
+
monday: *daily_schedule
|
47
|
+
tuesday: *daily_schedule
|
48
|
+
wednesday: *daily_schedule
|
49
|
+
thursday: *daily_schedule
|
50
|
+
friday: *daily_schedule
|
51
|
+
|
52
|
+
_default: &default
|
53
|
+
wait_iterations: 600 # Optional
|
54
|
+
command_log_lines: 100 # Optional
|
55
|
+
autoscale_type: ~ # Optional
|
56
|
+
region: us-east-1
|
57
|
+
app_name: 'wikiposit'
|
58
|
+
instance_type: 't2.small'
|
59
|
+
max_instance_duration: 86400 # Optional
|
60
|
+
clean_commands_to_ignore: ['configure', 'shutdown] # Optional: A list of opsworks commands to ignore when calculating that last run time for the clean command. Ignores `configure` and `shutdown` commands by default.
|
61
|
+
cookbook_dir: cookbooks
|
62
|
+
cookbook_name: wikiposit
|
63
|
+
cookbook_version: "<%= IO.read(File.join(Dir.pwd, 'cookbooks/VERSION')).strip %>"
|
64
|
+
cookbook_s3_bucket: quandl-cookbooks
|
65
|
+
|
66
|
+
staging:
|
67
|
+
<<: *default
|
68
|
+
deploy_type: :staging
|
69
|
+
stack_id: 1aec9354-e1bc-4f31-8627-2208f2382dcb
|
70
|
+
layer_id: 622555c4-fc07-4ff1-ba79-0a63fbd233f5
|
71
|
+
application_id: 15904509-ace3-4d78-923e-ea10b3b2d433 # Optional. Deploy command will not run without application ID.
|
72
|
+
subnet: subnet-0cde5d27
|
73
|
+
cookbook_s3_path: staging/my-app
|
74
|
+
```
|
75
|
+
|
76
|
+
## Sample Slack.yml
|
77
|
+
|
78
|
+
If you create a `config/quandl/slack.yml` file as so slack messages can be enabled for deployments under various environments.
|
79
|
+
|
80
|
+
```
|
81
|
+
defaults: &defaults
|
82
|
+
webhook_url: https://hooks.slack.com/services/......
|
83
|
+
notifiers: &default_notifiers
|
84
|
+
cookbook:
|
85
|
+
channel: '#releases'
|
86
|
+
username: My App Cookbooks
|
87
|
+
icon_emoji: ':book:'
|
88
|
+
release:
|
89
|
+
channel: '#releases'
|
90
|
+
username: My App
|
91
|
+
icon_emoji: ':rocket:'
|
92
|
+
instance_up:
|
93
|
+
channel: '#releases'
|
94
|
+
username: My App
|
95
|
+
icon_emoji: ':chart_with_upwards_trend:'
|
96
|
+
instance_down:
|
97
|
+
channel: '#releases'
|
98
|
+
username: My App
|
99
|
+
icon_emoji: ':chart_with_downwards_trend:'
|
100
|
+
|
101
|
+
development:
|
102
|
+
<<: *defaults
|
103
|
+
```
|
data/bin/qops
ADDED
data/lib/qops.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'thor/group'
|
3
|
+
require 'aws-sdk'
|
4
|
+
require 'json'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'active_support/all'
|
7
|
+
require 'pp'
|
8
|
+
require 'optparse'
|
9
|
+
require 'erb'
|
10
|
+
require 'rainbow'
|
11
|
+
|
12
|
+
require 'quandl/slack'
|
13
|
+
|
14
|
+
require_relative 'qops/environment'
|
15
|
+
require_relative 'qops/helpers'
|
16
|
+
require_relative 'qops/deployment/helpers'
|
17
|
+
require_relative 'qops/deployment/app'
|
18
|
+
require_relative 'qops/deployment/instances'
|
19
|
+
require_relative 'qops/cookbook/cookbook'
|
20
|
+
|
21
|
+
# Migrate this into quandl config project
|
22
|
+
module Quandl
|
23
|
+
class Config < ::OpenStruct
|
24
|
+
cattr_accessor :environment
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def project_environment
|
29
|
+
@_environment ||= @@environment || (defined?(Rails) ? ::Rails.env : nil) || ENV['RAILS_ENV'] || ENV['RAKE_ENV'] || ENV['QUANDL_ENV'] || 'default'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,160 @@
|
|
1
|
+
class Qops::Cookbook < Thor
|
2
|
+
include Qops::Helpers
|
3
|
+
|
4
|
+
desc 'vendor', 'Generate vendor directory to contain the cookbooks'
|
5
|
+
def vendor
|
6
|
+
initialize_run
|
7
|
+
cleanup
|
8
|
+
Dir.chdir(config.cookbook_dir) do
|
9
|
+
system('berks vendor vendor -e opsworks')
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
desc 'package', 'Package the cookbooks into a zip file in vendor'
|
14
|
+
def package
|
15
|
+
initialize_run
|
16
|
+
Dir.chdir(config.cookbook_dir) do
|
17
|
+
remove_zip_files
|
18
|
+
system("zip -r #{artifact_name} vendor/*")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
desc 'upload', 'Uploads cookbooks to s3'
|
23
|
+
def upload
|
24
|
+
initialize_run
|
25
|
+
s3.put_object(
|
26
|
+
bucket: config.cookbook_s3_bucket,
|
27
|
+
acl: 'private',
|
28
|
+
key: remote_artifact_file,
|
29
|
+
body: IO.read(local_artifact_file)
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'update_custom_json', 'Upload custom json to stack'
|
34
|
+
def update_custom_json
|
35
|
+
initialize_run
|
36
|
+
raw_json = File.read(File.join(config.cookbook_dir, config.cookbook_json))
|
37
|
+
json = JSON.parse(raw_json)
|
38
|
+
|
39
|
+
say(JSON.pretty_generate(json), :yellow)
|
40
|
+
if yes?("Are you sure you want to update the custom JSON for opsworks stack #{config.stack_id}?", :yellow)
|
41
|
+
config.opsworks.update_stack(
|
42
|
+
stack_id: config.stack_id,
|
43
|
+
custom_json: JSON.pretty_generate(json)
|
44
|
+
)
|
45
|
+
say('Updated!', :green)
|
46
|
+
else
|
47
|
+
say('You said no, so we\'re done here.', :yellow)
|
48
|
+
end
|
49
|
+
rescue JSON::ParserError
|
50
|
+
say('Check your JSON for errors!', :red)
|
51
|
+
end
|
52
|
+
|
53
|
+
desc 'update_stack_cookbooks', 'Runs the opsworks command to update custom cookbooks.'
|
54
|
+
def update_stack_cookbooks
|
55
|
+
initialize_run
|
56
|
+
if yes?("Are you sure you want to run the 'Update Custom Cookbooks' command on stack #{config.stack_id}?", :yellow)
|
57
|
+
run_opsworks_command(
|
58
|
+
stack_id: config.stack_id,
|
59
|
+
command: {
|
60
|
+
name: 'update_custom_cookbooks'
|
61
|
+
}
|
62
|
+
)
|
63
|
+
say('Updated!', :green)
|
64
|
+
else
|
65
|
+
say('You said no, so we\'re done here.', :yellow)
|
66
|
+
exit(-1)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
desc 'release', 'Zip, package and update a new cookbook as a release.'
|
71
|
+
def release
|
72
|
+
vendor && package && upload && update_custom_cookbooks && update_stack_cookbooks
|
73
|
+
|
74
|
+
ping_slack('Quandl::Slack::Cookbook', 'Cookbook updated', 'success',
|
75
|
+
command: 'opsworks cookbook release',
|
76
|
+
status: 'success',
|
77
|
+
name: config.cookbook_name,
|
78
|
+
version: config.cookbook_version,
|
79
|
+
stack: config.stack_id
|
80
|
+
)
|
81
|
+
|
82
|
+
say('Released!', :green)
|
83
|
+
end
|
84
|
+
|
85
|
+
desc 'update_custom_cookbooks', 'Update the stack with the custom cookbooks!'
|
86
|
+
def update_custom_cookbooks
|
87
|
+
if yes?("Are you sure you want to update the custom the custom cookbook for opsworks stack #{config.stack_id}?", :yellow)
|
88
|
+
config.opsworks.update_stack(
|
89
|
+
stack_id: config.stack_id,
|
90
|
+
use_custom_cookbooks: true,
|
91
|
+
custom_cookbooks_source: {
|
92
|
+
type: 's3',
|
93
|
+
url: "https://s3.amazonaws.com/#{config.cookbook_s3_bucket}/#{remote_artifact_file}"
|
94
|
+
})
|
95
|
+
say('Cookbooks updated', :green)
|
96
|
+
else
|
97
|
+
say('You said no, so we\'re done here.', :yellow)
|
98
|
+
exit(-1)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
desc 'cleanup', 'Cleanup all temporary files from the cookbook directory'
|
103
|
+
def cleanup
|
104
|
+
Dir.chdir(config.cookbook_dir) do
|
105
|
+
remove_zip_files
|
106
|
+
FileUtils.remove_dir('vendor') if File.directory?('vendor')
|
107
|
+
say("Cleaned up directory '#{config.cookbook_dir}/vendor'", :green)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
def initialize_run
|
114
|
+
super
|
115
|
+
config
|
116
|
+
end
|
117
|
+
|
118
|
+
def config
|
119
|
+
return @_config if @_config
|
120
|
+
|
121
|
+
@_config ||= Qops::Environment.new
|
122
|
+
|
123
|
+
%w(cookbook_dir cookbook_s3_bucket cookbook_s3_path cookbook_name cookbook_version).each do |var|
|
124
|
+
fail ArgumentError.new("Must specify a '#{var}' in the config") if !@_config.respond_to?(var) && !@_config.configuration.respond_to?(var)
|
125
|
+
end
|
126
|
+
|
127
|
+
fail ArgumentError.new("Cannot find/do not have access to cookbook directory: #{@_config.cookbook_dir}") unless Dir.exist?(@_config.cookbook_dir)
|
128
|
+
|
129
|
+
@_config
|
130
|
+
end
|
131
|
+
|
132
|
+
def s3
|
133
|
+
@s3 ||= Aws::S3::Client.new(
|
134
|
+
region: 'us-east-1',
|
135
|
+
access_key_id: config.opsworks.config.credentials.access_key_id,
|
136
|
+
secret_access_key: config.opsworks.config.credentials.secret_access_key
|
137
|
+
)
|
138
|
+
end
|
139
|
+
|
140
|
+
def local_artifact_file
|
141
|
+
File.join(config.cookbook_dir, artifact_name)
|
142
|
+
end
|
143
|
+
|
144
|
+
def remote_artifact_file
|
145
|
+
File.join(config.cookbook_s3_path, artifact_name)
|
146
|
+
end
|
147
|
+
|
148
|
+
def artifact_name
|
149
|
+
"#{config.cookbook_name}-#{config.cookbook_version}.zip"
|
150
|
+
end
|
151
|
+
|
152
|
+
def vendor_dir
|
153
|
+
File.join(config.cookbook_dir, 'vendor')
|
154
|
+
end
|
155
|
+
|
156
|
+
def remove_zip_files
|
157
|
+
FileUtils.rm Dir.glob("#{config.cookbook_name}*.zip")
|
158
|
+
say("Cleaned up directory '#{config.cookbook_dir}/*.zip'", :green)
|
159
|
+
end
|
160
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
class Qops::Deploy < Thor
|
2
|
+
include Qops::DeployHelpers
|
3
|
+
|
4
|
+
class_option :custom_json, type: :string, aliases: '-j', desc: 'A custom json that will be used during a deployment of the app. ex: \'{ "custom_attrs": "are awesome!"}\''
|
5
|
+
|
6
|
+
desc 'app', 'Deploy the latest version of the app'
|
7
|
+
def app
|
8
|
+
initialize_run
|
9
|
+
|
10
|
+
instances = if config.deploy_type == 'staging'
|
11
|
+
[retrieve_instance].compact
|
12
|
+
else
|
13
|
+
retrieve_instances
|
14
|
+
end
|
15
|
+
online_instances = instances.select { |instance| instance.status == 'online' }
|
16
|
+
|
17
|
+
if online_instances.count == 0
|
18
|
+
raise 'Could not find any running instance(s) to deploy to. Perhaps you need to run "qops:instance:up" first'
|
19
|
+
end
|
20
|
+
|
21
|
+
if config.deploy_type == 'staging'
|
22
|
+
puts "Preparing to deploy branch #{default_revision} to instance #{online_instances.first.hostname}"
|
23
|
+
else
|
24
|
+
puts "Preparing to deploy default branch to all (online) servers (#{online_instances.map(&:hostname).join(', ')})"
|
25
|
+
end
|
26
|
+
|
27
|
+
base_deployment_params = {
|
28
|
+
stack_id: config.stack_id,
|
29
|
+
command: { name: 'deploy' }
|
30
|
+
}
|
31
|
+
|
32
|
+
if !config.application_id
|
33
|
+
Qops::Environment.print_with_colour('No application specified. Exiting without application deployment.')
|
34
|
+
exit(0)
|
35
|
+
else
|
36
|
+
base_deployment_params[:app_id] = config.application_id
|
37
|
+
end
|
38
|
+
|
39
|
+
if config.deploy_type != 'production'
|
40
|
+
base_deployment_params[:custom_json] = custom_json.to_json
|
41
|
+
end
|
42
|
+
|
43
|
+
manifest = { environment: config.deploy_type }
|
44
|
+
|
45
|
+
# Deploy the first instance with migration on
|
46
|
+
first_instance = online_instances.first
|
47
|
+
print "Migrating and deploying first instance (#{first_instance.hostname}) ..."
|
48
|
+
deployment_params = base_deployment_params.deep_dup
|
49
|
+
should_migrate = !config.option?(:migrate) || config.option?(:migrate) && config.migrate == true
|
50
|
+
deployment_params[:command][:args] = { migrate: ['true'] } if should_migrate
|
51
|
+
run_opsworks_command(deployment_params, [first_instance.instance_id])
|
52
|
+
ping_slack(
|
53
|
+
'Quandl::Slack::Release',
|
54
|
+
"Deployed and migrated instance '#{first_instance.hostname}'",
|
55
|
+
'success',
|
56
|
+
manifest.merge(
|
57
|
+
app_name: config.app_name,
|
58
|
+
command: 'deploy + migrate',
|
59
|
+
migrate: should_migrate.to_s,
|
60
|
+
completed: Time.now,
|
61
|
+
hostname: first_instance.hostname,
|
62
|
+
instance_id: first_instance.instance_id
|
63
|
+
)
|
64
|
+
)
|
65
|
+
|
66
|
+
# Deploy any remaining instances with migration off for production
|
67
|
+
return unless config.deploy_type == 'production' && online_instances.count > 1
|
68
|
+
|
69
|
+
print 'Deploying remaining instances ...'
|
70
|
+
deployment_params = base_deployment_params.deep_dup
|
71
|
+
run_opsworks_command(deployment_params)
|
72
|
+
ping_slack(
|
73
|
+
'Quandl::Slack::Release',
|
74
|
+
'Deployed All Instances',
|
75
|
+
'success',
|
76
|
+
manifest.merge(
|
77
|
+
app_name: config.app_name,
|
78
|
+
command: 'deploy',
|
79
|
+
migrate: false,
|
80
|
+
completed: Time.now,
|
81
|
+
hostname: online_instances.map(&:hostname),
|
82
|
+
instance_id: online_instances.map(&:instance_id)
|
83
|
+
)
|
84
|
+
)
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def custom_json
|
90
|
+
return @_custom_json if @_custom_json
|
91
|
+
|
92
|
+
@_custom_json = {}
|
93
|
+
|
94
|
+
if config.application_id
|
95
|
+
application_name = config.opsworks.describe_apps(app_ids: [config.application_id]).apps.first.name
|
96
|
+
|
97
|
+
@_custom_json[:deploy] = {
|
98
|
+
application_name => {
|
99
|
+
scm: {
|
100
|
+
revision: default_revision
|
101
|
+
}
|
102
|
+
}
|
103
|
+
}
|
104
|
+
end
|
105
|
+
|
106
|
+
if options[:custom_json].present?
|
107
|
+
your_json = JSON.parse(options[:custom_json])
|
108
|
+
@_custom_json.merge!(your_json)
|
109
|
+
puts "Using custom json:\n#{JSON.pretty_generate(@_custom_json)}"
|
110
|
+
end
|
111
|
+
|
112
|
+
@_custom_json
|
113
|
+
rescue JSON::ParserError
|
114
|
+
say('Your custom json has invalid syntax. Failing ...', :red)
|
115
|
+
exit(-1)
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Qops::DeployHelpers
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
include Qops::Helpers
|
5
|
+
|
6
|
+
included do
|
7
|
+
class_option :branch, type: :string, aliases: '-b', desc: 'The branch to use when deploying to staging type environments'
|
8
|
+
end
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def config
|
13
|
+
return @_config if @_config
|
14
|
+
|
15
|
+
Qops::Environment.notifiers
|
16
|
+
@_config ||= Qops::Environment.new
|
17
|
+
|
18
|
+
fail "Invalid configure deploy_type detected: #{@_config.deploy_type}" unless %w(staging production).include?(@_config.deploy_type)
|
19
|
+
|
20
|
+
@_config
|
21
|
+
end
|
22
|
+
|
23
|
+
def retrieve_instances(options = {})
|
24
|
+
# Describe and create instances as necessary
|
25
|
+
instances_results = config.opsworks.describe_instances({ layer_id: config.layer_id }.merge(options))
|
26
|
+
|
27
|
+
# Determine if instance exists.
|
28
|
+
instances_results.data.instances
|
29
|
+
end
|
30
|
+
|
31
|
+
def retrieve_instance(instance_id = nil)
|
32
|
+
# Retrieve a specific instance as necessary
|
33
|
+
if instance_id
|
34
|
+
instances_results = config.opsworks.describe_instances(instance_ids: [instance_id])
|
35
|
+
return instances_results.data.instances.first
|
36
|
+
end
|
37
|
+
|
38
|
+
# Get instance based on hostname
|
39
|
+
instances_results = config.opsworks.describe_instances(layer_id: config.layer_id)
|
40
|
+
|
41
|
+
# Determine if instance exists.
|
42
|
+
instances = instances_results.data.instances
|
43
|
+
|
44
|
+
return unless instances.map(&:hostname).include?(requested_hostname)
|
45
|
+
|
46
|
+
instances.find { |k| k.hostname == requested_hostname }
|
47
|
+
end
|
48
|
+
|
49
|
+
def requested_hostname
|
50
|
+
return @requested_hostname if @requested_hostname
|
51
|
+
if config.deploy_type == 'staging'
|
52
|
+
@requested_hostname = default_revision.parameterize
|
53
|
+
elsif config.deploy_type == 'production'
|
54
|
+
@requested_hostname = config.app_name
|
55
|
+
existing_hostnames = retrieve_instances.map(&:hostname)
|
56
|
+
@requested_hostname += "-#{existing_hostnames.sort.last.to_s.split('-').last.to_i + 1}"
|
57
|
+
end
|
58
|
+
|
59
|
+
@requested_hostname = @requested_hostname.gsub(/[^A-Za-z0-9\-]+/, '-').gsub(/-+/, '-')
|
60
|
+
@requested_hostname = @requested_hostname[0..62]
|
61
|
+
@requested_hostname = @requested_hostname.match(/^([A-Za-z0-9\-]+).*$/)[1]
|
62
|
+
end
|
63
|
+
|
64
|
+
def default_revision
|
65
|
+
return 'master' unless config.deploy_type == 'staging'
|
66
|
+
if options[:branch].present?
|
67
|
+
options[:branch]
|
68
|
+
elsif `git --version` # rubocop:disable Lint/LiteralInCondition
|
69
|
+
`git symbolic-ref --short HEAD`.strip
|
70
|
+
else
|
71
|
+
'master'
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,351 @@
|
|
1
|
+
class Qops::Instance < Thor # rubocop:disable Metrics/ClassLength
|
2
|
+
include Qops::DeployHelpers
|
3
|
+
|
4
|
+
desc 'up', 'Deploy the current branch to new or existing instance(s)'
|
5
|
+
def up
|
6
|
+
initialize_run
|
7
|
+
|
8
|
+
# Get the instance(s) to work with if they exist. In production we always create a new instacne
|
9
|
+
instance = retrieve_instance if config.deploy_type == 'staging'
|
10
|
+
|
11
|
+
# Create the instance if necessary
|
12
|
+
if instance
|
13
|
+
instance_id = instance.instance_id
|
14
|
+
puts "Existing instance #{requested_hostname}"
|
15
|
+
else
|
16
|
+
params = {
|
17
|
+
stack_id: config.stack_id,
|
18
|
+
layer_ids: [config.layer_id],
|
19
|
+
instance_type: config.instance_type,
|
20
|
+
os: 'Ubuntu 14.04 LTS',
|
21
|
+
hostname: requested_hostname,
|
22
|
+
subnet_id: config.subnet,
|
23
|
+
auto_scaling_type: config.autoscale_type,
|
24
|
+
architecture: 'x86_64',
|
25
|
+
root_device_type: 'ebs',
|
26
|
+
block_device_mappings: [
|
27
|
+
{
|
28
|
+
device_name: 'ROOT_DEVICE',
|
29
|
+
ebs: {
|
30
|
+
volume_size: config.root_volume_size,
|
31
|
+
volume_type: 'gp2',
|
32
|
+
delete_on_termination: true
|
33
|
+
}
|
34
|
+
}
|
35
|
+
],
|
36
|
+
ebs_optimized: config.deploy_type =~ /production/
|
37
|
+
}
|
38
|
+
puts 'Creating instance with params: ' + params.inspect
|
39
|
+
instance_id = config.opsworks.create_instance(params).data.instance_id
|
40
|
+
creating_instance = true
|
41
|
+
end
|
42
|
+
|
43
|
+
instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
|
44
|
+
instance = instance_results.data.instances.first
|
45
|
+
|
46
|
+
# Set up the automatic boot scheduler
|
47
|
+
if config.autoscale_type == 'timer'
|
48
|
+
print 'Setting up weekly schedule ...'
|
49
|
+
config.opsworks.set_time_based_auto_scaling(instance_id: instance_id, auto_scaling_schedule: config.schedule)
|
50
|
+
print "done\n"
|
51
|
+
end
|
52
|
+
|
53
|
+
# Record the initial instance before doing anything.
|
54
|
+
initial_instance_state = instance
|
55
|
+
|
56
|
+
# Start the instance if necessary
|
57
|
+
print 'Booting instance ...'
|
58
|
+
unless %w(online booting).include?(instance.status)
|
59
|
+
config.opsworks.start_instance(instance_id: instance_id)
|
60
|
+
end
|
61
|
+
|
62
|
+
manifest = {
|
63
|
+
environment: config.deploy_type,
|
64
|
+
app_name: config.app_name,
|
65
|
+
command: 'add instance'
|
66
|
+
}
|
67
|
+
|
68
|
+
# Boot the instance
|
69
|
+
iterator(manifest) do |i|
|
70
|
+
instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
|
71
|
+
instance = instance_results.data.instances.first
|
72
|
+
|
73
|
+
if %w(booting requested pending).include?(instance.status)
|
74
|
+
print '.'
|
75
|
+
print " #{instance.status} :" if been_a_minute?(i)
|
76
|
+
else
|
77
|
+
puts ' ' + instance.status
|
78
|
+
true
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
puts "Public IP: #{instance.public_ip}"
|
83
|
+
puts "Private IP: #{instance.private_ip}"
|
84
|
+
|
85
|
+
setup_instance(instance, initial_instance_state, manifest)
|
86
|
+
|
87
|
+
if creating_instance
|
88
|
+
ping_slack(
|
89
|
+
'Quandl::Slack::InstanceUp',
|
90
|
+
'Created another instance',
|
91
|
+
'success',
|
92
|
+
manifest.merge(
|
93
|
+
completed: Time.now,
|
94
|
+
hostname: instance.hostname,
|
95
|
+
instance_id: instance.instance_id,
|
96
|
+
private_ip: instance.private_ip,
|
97
|
+
public_ip: instance.public_ip.blank? ? 'N/A' : instance.public_ip
|
98
|
+
)
|
99
|
+
)
|
100
|
+
end
|
101
|
+
|
102
|
+
# For Elasticsearch cluster, register with public elb
|
103
|
+
if config.option?(:public_search_elb)
|
104
|
+
print "Register instance #{instance.ec2_instance_id} to elb #{config.public_search_elb}"
|
105
|
+
elb.register_instances_with_load_balancer(load_balancer_name: config.public_search_elb.to_s,
|
106
|
+
instances: [{ instance_id: instance.ec2_instance_id.to_s }])
|
107
|
+
end
|
108
|
+
|
109
|
+
# Deploy the latest code to instance
|
110
|
+
Qops::Deploy.new([], options).app
|
111
|
+
end
|
112
|
+
|
113
|
+
desc 'down', 'Remove the instance associated with the given branch'
|
114
|
+
def down
|
115
|
+
initialize_run
|
116
|
+
|
117
|
+
# Get the instance to shutdown
|
118
|
+
if config.deploy_type == 'staging'
|
119
|
+
instance = retrieve_instance
|
120
|
+
elsif config.deploy_type == 'production'
|
121
|
+
instance = retrieve_instances.first
|
122
|
+
end
|
123
|
+
|
124
|
+
if instance.nil?
|
125
|
+
puts 'No instance available to shutdown'
|
126
|
+
exit(0)
|
127
|
+
else
|
128
|
+
instance_id = instance.instance_id
|
129
|
+
end
|
130
|
+
|
131
|
+
terminate_instance(instance_id)
|
132
|
+
|
133
|
+
puts 'Success'
|
134
|
+
end
|
135
|
+
|
136
|
+
desc 'rebuild', 'Runs the down then up command to rebuild an instance.'
|
137
|
+
def rebuild
|
138
|
+
down
|
139
|
+
up
|
140
|
+
end
|
141
|
+
|
142
|
+
desc 'clean', 'Cleans up old instances in staging type environments'
|
143
|
+
def clean
|
144
|
+
initialize_run
|
145
|
+
|
146
|
+
if config.deploy_type == 'production'
|
147
|
+
fail "Cannot clean instances in a #{config.deploy_type} environment"
|
148
|
+
end
|
149
|
+
|
150
|
+
terminated_instances = []
|
151
|
+
|
152
|
+
# Find all instances to be destroyed
|
153
|
+
retrieve_instances.each do |instance|
|
154
|
+
next if instance.hostname == 'master'
|
155
|
+
|
156
|
+
# Find the latest command since the instance was deployed
|
157
|
+
latest_command = Time.parse(instance.created_at)
|
158
|
+
config.opsworks.describe_commands(instance_id: instance.instance_id).commands.each do |command|
|
159
|
+
next if config.clean_commands_to_ignore.include?(command.type)
|
160
|
+
completed_at = Time.parse(command.completed_at || command.acknowledged_at || command.created_at)
|
161
|
+
latest_command = completed_at if completed_at > latest_command
|
162
|
+
end
|
163
|
+
|
164
|
+
# If the latest deployment is greater than the maximum alive time allowed remove the instance.
|
165
|
+
if Time.now.to_i - latest_command.to_i > config.max_instance_duration
|
166
|
+
terminate_instance(instance.instance_id)
|
167
|
+
terminated_instances << instance
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
if terminated_instances.any?
|
172
|
+
puts "Terminated instances: #{terminated_instances.map(&:hostname).join("\n")}"
|
173
|
+
else
|
174
|
+
puts 'No unused instances old enough to terminate.'
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
desc 'run_command', 'Run command on existing instance(s) at once or each one by one'
|
179
|
+
def run_command
|
180
|
+
initialize_run
|
181
|
+
instances = retrieve_instances
|
182
|
+
|
183
|
+
puts "Preparing to run command to all servers (#{instances.map(&:hostname).join(', ')})"
|
184
|
+
|
185
|
+
command = ask('Which command you want to execute?', limited_to: %w(setup configure install_dependencies update_dependencies))
|
186
|
+
|
187
|
+
option = ask('Which command you want to execute?', limited_to: %w(all_in_once one_by_one))
|
188
|
+
|
189
|
+
base_deployment_params = {
|
190
|
+
stack_id: config.stack_id,
|
191
|
+
command: { name: command.to_s }
|
192
|
+
}
|
193
|
+
|
194
|
+
manifest = { environment: config.deploy_type }
|
195
|
+
|
196
|
+
case option
|
197
|
+
when 'all_in_once'
|
198
|
+
print "Run command #{command} on all instances at once ..."
|
199
|
+
deployment_params = base_deployment_params.deep_dup
|
200
|
+
run_opsworks_command(deployment_params)
|
201
|
+
ping_slack(
|
202
|
+
'Quandl::Slack::Release',
|
203
|
+
"Run command: `#{command}` on all instances",
|
204
|
+
'success',
|
205
|
+
manifest.merge(
|
206
|
+
app_name: config.app_name,
|
207
|
+
command: 'deploy',
|
208
|
+
migrate: false,
|
209
|
+
completed: Time.now,
|
210
|
+
hostname: instances.map(&:hostname),
|
211
|
+
instance_id: instances.map(&:instance_id)
|
212
|
+
)
|
213
|
+
)
|
214
|
+
else
|
215
|
+
instances.each do |instance|
|
216
|
+
print "Run command #{command} on instance #{instance.ec2_instance_id}"
|
217
|
+
|
218
|
+
run_opsworks_command(base_deployment_params, [instance.instance_id])
|
219
|
+
|
220
|
+
ping_slack('Quandl::Slack::InstanceDown', "Run command: `#{command}` on existing instance", 'success',
|
221
|
+
manifest.merge(
|
222
|
+
completed: Time.now,
|
223
|
+
hostname: instance.hostname,
|
224
|
+
instance_id: instance.instance_id,
|
225
|
+
private_ip: instance.private_ip,
|
226
|
+
public_ip: instance.public_ip
|
227
|
+
)
|
228
|
+
)
|
229
|
+
puts 'Success'
|
230
|
+
break if instance.instance_id == instances.last.instance_id
|
231
|
+
delay = config.wait_deploy
|
232
|
+
puts "wait for #{delay / 60.0} mintues"
|
233
|
+
sleep delay
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
private
|
239
|
+
|
240
|
+
def elb
|
241
|
+
@elb ||= Aws::ElasticLoadBalancing::Client.new(
|
242
|
+
region: 'us-east-1',
|
243
|
+
access_key_id: config.opsworks.config.credentials.access_key_id,
|
244
|
+
secret_access_key: config.opsworks.config.credentials.secret_access_key)
|
245
|
+
end
|
246
|
+
|
247
|
+
def setup_instance(instance, initial_instance_state, manifest)
|
248
|
+
# If the previous instance setup failed then run the setup task again when trying to bring up the instance.
|
249
|
+
if initial_instance_state.status == 'setup_failed'
|
250
|
+
print 'Setup instance ...'
|
251
|
+
run_opsworks_command(
|
252
|
+
{
|
253
|
+
stack_id: config.stack_id,
|
254
|
+
command: {
|
255
|
+
name: 'setup'
|
256
|
+
}
|
257
|
+
},
|
258
|
+
[instance.instance_id]
|
259
|
+
)
|
260
|
+
|
261
|
+
# Monitor the existing instance setup.
|
262
|
+
else
|
263
|
+
print 'Setup instance ...'
|
264
|
+
iterator(manifest) do |i|
|
265
|
+
instance_results = config.opsworks.describe_instances(instance_ids: [instance.instance_id])
|
266
|
+
instance = instance_results.data.instances.first
|
267
|
+
|
268
|
+
if %w(online).include?(instance.status)
|
269
|
+
puts ' ' + instance.status
|
270
|
+
true
|
271
|
+
elsif %w(setup_failed).include?(instance.status)
|
272
|
+
puts ' ' + instance.status
|
273
|
+
read_failure_log(
|
274
|
+
{ instance_id: instance.instance_id },
|
275
|
+
last_only: true,
|
276
|
+
manifest: manifest.merge(
|
277
|
+
hostname: instance.hostname,
|
278
|
+
instance_id: instance.instance_id,
|
279
|
+
private_ip: instance.private_ip,
|
280
|
+
public_ip: instance.public_ip
|
281
|
+
)
|
282
|
+
)
|
283
|
+
exit(-1)
|
284
|
+
else
|
285
|
+
print '.'
|
286
|
+
print " #{instance.status} :" if been_a_minute?(i)
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
def terminate_instance(instance_id)
|
293
|
+
# Remove schedule if time based instance
|
294
|
+
if config.autoscale_type == 'timer'
|
295
|
+
config.opsworks.set_time_based_auto_scaling(instance_id: instance_id, auto_scaling_schedule: {})
|
296
|
+
end
|
297
|
+
|
298
|
+
# Get the instance from the id
|
299
|
+
instance = retrieve_instance(instance_id)
|
300
|
+
|
301
|
+
# For Elasticsearch cluster, remove from from public elb
|
302
|
+
if config.option?(:public_search_elb)
|
303
|
+
elb.deregister_instances_from_load_balancer(
|
304
|
+
load_balancer_name: config.public_search_elb.to_s,
|
305
|
+
instances: [{ instance_id: instance.ec2_instance_id.to_s }]
|
306
|
+
)
|
307
|
+
end
|
308
|
+
|
309
|
+
# Attempt to shutdown the instance
|
310
|
+
print "Attempting instance #{instance_id}-#{instance.hostname} shutdown ..."
|
311
|
+
unless instance.status == 'stopped'
|
312
|
+
config.opsworks.stop_instance(instance_id: instance_id)
|
313
|
+
end
|
314
|
+
|
315
|
+
manifest = {
|
316
|
+
environment: config.deploy_type,
|
317
|
+
app_name: config.app_name,
|
318
|
+
command: 'remove instance'
|
319
|
+
}
|
320
|
+
|
321
|
+
iterator(manifest) do |i|
|
322
|
+
instance_results = config.opsworks.describe_instances(instance_ids: [instance_id])
|
323
|
+
instance = instance_results.data.instances.first
|
324
|
+
|
325
|
+
if instance.status == 'stopped'
|
326
|
+
puts ' ' + instance.status
|
327
|
+
true
|
328
|
+
else
|
329
|
+
print '.'
|
330
|
+
print " #{instance.status} :" if been_a_minute?(i)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
# Terminate the instance
|
335
|
+
puts "Terminating instance #{instance_id}"
|
336
|
+
config.opsworks.delete_instance(instance_id: instance_id, delete_volumes: true)
|
337
|
+
|
338
|
+
ping_slack(
|
339
|
+
'Quandl::Slack::InstanceDown',
|
340
|
+
'Remove existing instance',
|
341
|
+
'success',
|
342
|
+
manifest.merge(
|
343
|
+
completed: Time.now,
|
344
|
+
hostname: instance.hostname,
|
345
|
+
instance_id: instance.instance_id,
|
346
|
+
private_ip: instance.private_ip,
|
347
|
+
public_ip: instance.public_ip
|
348
|
+
)
|
349
|
+
)
|
350
|
+
end
|
351
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'quandl/config'
|
2
|
+
|
3
|
+
module Qops
|
4
|
+
class Environment
|
5
|
+
include Quandl::Configurable
|
6
|
+
|
7
|
+
def self.file_name
|
8
|
+
'opsworks'
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.notifiers
|
12
|
+
return @_notifiers unless @_notifiers.nil?
|
13
|
+
if File.exist?('config/quandl/slack.yml')
|
14
|
+
@_notifiers ||= Quandl::Slack.autogenerate_notifiers
|
15
|
+
else
|
16
|
+
@_notifiers = false
|
17
|
+
print_with_colour('Slack notifications disabled. Could not find slack configuration at: config/quandl/slack.yml', :warning)
|
18
|
+
end
|
19
|
+
rescue NoMethodError => e
|
20
|
+
print_with_colour("Slack notifications disabled due to an error. #{e}", :warning)
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.print_with_colour(message, level = :normal)
|
24
|
+
case level
|
25
|
+
when :error
|
26
|
+
puts Rainbow(message).bg(:black).red
|
27
|
+
when :warning
|
28
|
+
puts Rainbow(message).bg(:black).yellow
|
29
|
+
else
|
30
|
+
puts message
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def initialize
|
35
|
+
%w(deploy_type region stack_id app_name).each do |v|
|
36
|
+
fail "Please configure #{v} before continuing." unless option?(v)
|
37
|
+
end
|
38
|
+
|
39
|
+
begin
|
40
|
+
opsworks.config.credentials.access_key_id
|
41
|
+
opsworks.config.credentials.secret_access_key
|
42
|
+
rescue => e
|
43
|
+
raise "There may be a problem with your aws credentials. Please correct with `aws configure`. Error: #{e}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def application_id
|
48
|
+
configuration.application_id unless configuration.application_id.blank?
|
49
|
+
end
|
50
|
+
|
51
|
+
def deploy_type
|
52
|
+
configuration.deploy_type.to_s
|
53
|
+
end
|
54
|
+
|
55
|
+
def command_log_lines
|
56
|
+
configuration.command_log_lines || 100
|
57
|
+
end
|
58
|
+
|
59
|
+
def wait_iterations
|
60
|
+
configuration.wait_iterations || 600
|
61
|
+
end
|
62
|
+
|
63
|
+
def wait_deploy
|
64
|
+
configuration.wait_deploy || 180
|
65
|
+
end
|
66
|
+
|
67
|
+
def autoscale_type
|
68
|
+
configuration.autoscale_type || nil
|
69
|
+
end
|
70
|
+
|
71
|
+
# Default 1 days
|
72
|
+
def max_instance_duration
|
73
|
+
configuration.max_instance_duration || 86_400
|
74
|
+
end
|
75
|
+
|
76
|
+
def clean_commands_to_ignore
|
77
|
+
configuration.clean_commands_to_ignore.present? ? configuration.clean_commands_to_ignore : %w(configure shutdown)
|
78
|
+
end
|
79
|
+
|
80
|
+
def file_name
|
81
|
+
self.class.file_name
|
82
|
+
end
|
83
|
+
|
84
|
+
def opsworks
|
85
|
+
@_opsworks ||= Aws::OpsWorks::Client.new(region: configuration.region)
|
86
|
+
end
|
87
|
+
|
88
|
+
def cookbook_json
|
89
|
+
configuration.cookbook_json || 'custom.json'
|
90
|
+
end
|
91
|
+
|
92
|
+
def option?(key)
|
93
|
+
respond_to?(key.to_sym) || configuration.instance_variable_get(:@table).keys.include?(key.to_sym)
|
94
|
+
end
|
95
|
+
|
96
|
+
def root_volume_size
|
97
|
+
configuration.root_volume_size || 30
|
98
|
+
end
|
99
|
+
|
100
|
+
private
|
101
|
+
|
102
|
+
def method_missing(method_sym, *arguments, &block)
|
103
|
+
if configuration.respond_to?(method_sym)
|
104
|
+
configuration.send(method_sym, *arguments, &block)
|
105
|
+
else
|
106
|
+
super
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
data/lib/qops/helpers.rb
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
module Qops::Helpers
|
2
|
+
extend ActiveSupport::Concern
|
3
|
+
|
4
|
+
included do
|
5
|
+
class_option :environment, aliases: '-e', desc: 'The environment to use when running commands.'
|
6
|
+
end
|
7
|
+
|
8
|
+
private
|
9
|
+
|
10
|
+
def initialize_run
|
11
|
+
return if @_run_initialized
|
12
|
+
|
13
|
+
@_run_initialized = true
|
14
|
+
verify_opsworks_config
|
15
|
+
verify_environment_selection
|
16
|
+
Qops::Environment.notifiers
|
17
|
+
end
|
18
|
+
|
19
|
+
def been_a_minute?(i)
|
20
|
+
i > 1 && i % 60 == 0
|
21
|
+
end
|
22
|
+
|
23
|
+
def iterator(options)
|
24
|
+
config.wait_iterations.times.each do |i|
|
25
|
+
result = yield(i)
|
26
|
+
break if result
|
27
|
+
|
28
|
+
if i + 1 == config.wait_iterations
|
29
|
+
puts " failed to complete within #{config.wait_iterations} seconds"
|
30
|
+
ping_slack('Quandl::Slack::Release', 'Command timeout', 'failure',
|
31
|
+
options.merge(failed_at: Time.now)
|
32
|
+
)
|
33
|
+
exit(-1)
|
34
|
+
elsif been_a_minute?(i)
|
35
|
+
print " #{i / 60} minute(s) "
|
36
|
+
end
|
37
|
+
|
38
|
+
sleep(1)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def ping_slack(notifier, message, status, manifest)
|
43
|
+
fields = manifest.keys
|
44
|
+
fields.map! { |field| { title: field, value: manifest[field], short: true } }
|
45
|
+
|
46
|
+
if Object.const_defined?(notifier)
|
47
|
+
notifier_class = notifier.constantize
|
48
|
+
|
49
|
+
notifier_class.ping("#{config.app_name}: #{message}",
|
50
|
+
attachments: [{ color: status, mrkdwn_in: ['text'], fallback: 'Details', fields: fields }]
|
51
|
+
)
|
52
|
+
else
|
53
|
+
puts "#{config.app_name}: #{message}"
|
54
|
+
pp fields
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def run_opsworks_command(deployment_params, instance_ids = [])
|
59
|
+
deployment_params[:instance_ids] = instance_ids unless instance_ids.empty?
|
60
|
+
|
61
|
+
# Create the deployment
|
62
|
+
deployment_results = config.opsworks.create_deployment(deployment_params)
|
63
|
+
deployment_id = deployment_results.data.deployment_id
|
64
|
+
|
65
|
+
iterator(deployment_params) do |i|
|
66
|
+
deployment_results = config.opsworks.describe_deployments(deployment_ids: [deployment_id])
|
67
|
+
deployment = deployment_results.data.deployments.first
|
68
|
+
|
69
|
+
if deployment.completed_at
|
70
|
+
puts ' ' + deployment.status
|
71
|
+
|
72
|
+
if deployment.status != 'successful'
|
73
|
+
read_failure_log(deployment_id: deployment.deployment_id)
|
74
|
+
exit(-1)
|
75
|
+
end
|
76
|
+
|
77
|
+
true
|
78
|
+
else
|
79
|
+
print '.'
|
80
|
+
if been_a_minute?(i)
|
81
|
+
print " #{retrieve_instance.status} :" if config.deploy_type == :staging && instance_ids.any?
|
82
|
+
print " #{deployment.status} :" if config.deploy_type == :production
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
print "\n"
|
88
|
+
end
|
89
|
+
|
90
|
+
def read_failure_log(opsworks_options, options = {})
|
91
|
+
results = config.opsworks.describe_commands(opsworks_options)
|
92
|
+
results.commands.each do |command|
|
93
|
+
if command.log_url
|
94
|
+
puts "\nReading last 100 lines from #{command.log_url}\n"
|
95
|
+
lines = open(command.log_url).read.split("\n")
|
96
|
+
num_lines = lines.count < config.command_log_lines ? lines.count : config.command_log_lines
|
97
|
+
puts open(command.log_url).read.split("\n")[-1 * num_lines..-1].join("\n")
|
98
|
+
puts "\nLog file at: #{command.log_url}"
|
99
|
+
end
|
100
|
+
|
101
|
+
ping_slack(
|
102
|
+
'Quandl::Slack::Release',
|
103
|
+
'Deployment failure',
|
104
|
+
'failure',
|
105
|
+
(options[:manifest] || {}).merge(
|
106
|
+
command: command.type,
|
107
|
+
status: command.status
|
108
|
+
)
|
109
|
+
)
|
110
|
+
|
111
|
+
exit(-1) if options[:last_only]
|
112
|
+
end
|
113
|
+
|
114
|
+
exit(-1)
|
115
|
+
end
|
116
|
+
|
117
|
+
def verify_opsworks_config
|
118
|
+
return if File.exist?("config/#{Qops::Environment.file_name}.yml")
|
119
|
+
raise "Could not find configuration file: config/#{Qops::Environment.file_name}.yml"
|
120
|
+
end
|
121
|
+
|
122
|
+
def verify_environment_selection
|
123
|
+
return if Quandl::Config.environment
|
124
|
+
|
125
|
+
project_root = Pathname.new(Quandl::ProjectRoot.root)
|
126
|
+
file_path = project_root.join('config', "#{Qops::Environment.file_name}.yml")
|
127
|
+
|
128
|
+
if File.exist?(file_path) # rubocop:disable Style/GuardClause
|
129
|
+
raw_config = File.read(file_path)
|
130
|
+
erb_config = ERB.new(raw_config).result
|
131
|
+
configs = YAML.load(erb_config)
|
132
|
+
|
133
|
+
env = options[:environment]
|
134
|
+
|
135
|
+
msg = 'Run command using config environment:'
|
136
|
+
msg = "Invalid config environment '#{env}'. Switch to:" if env && !configs.keys.include?(env)
|
137
|
+
|
138
|
+
unless env && configs.keys.include?(env)
|
139
|
+
env = Thor::Shell::Color.new.ask(
|
140
|
+
msg,
|
141
|
+
:yellow,
|
142
|
+
limited_to: configs.keys.reject { |g| g.start_with?('_') },
|
143
|
+
echo: false
|
144
|
+
)
|
145
|
+
end
|
146
|
+
|
147
|
+
Quandl::Config.environment = env
|
148
|
+
puts "\nRunning commands with config environment: #{env}"
|
149
|
+
else
|
150
|
+
raise "Not a qops compatible project. Please be sure to add a config/opsworks.yml file as described in the readme. Path: #{file_path}"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
data/lib/qops/version.rb
ADDED
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: qops
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matthew Basset
|
8
|
+
- Clemeny Leung
|
9
|
+
- Jason Byck
|
10
|
+
- Jun Li
|
11
|
+
autorequire:
|
12
|
+
bindir: bin
|
13
|
+
cert_chain: []
|
14
|
+
date: 2016-04-04 00:00:00.000000000 Z
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: thor
|
18
|
+
requirement: !ruby/object:Gem::Requirement
|
19
|
+
requirements:
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 0.19.1
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.19.1
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: aws-sdk
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
requirements:
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 2.0.41
|
37
|
+
type: :runtime
|
38
|
+
prerelease: false
|
39
|
+
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 2.0.41
|
44
|
+
- !ruby/object:Gem::Dependency
|
45
|
+
name: quandl-config
|
46
|
+
requirement: !ruby/object:Gem::Requirement
|
47
|
+
requirements:
|
48
|
+
- - ">="
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: 0.1.0
|
51
|
+
type: :runtime
|
52
|
+
prerelease: false
|
53
|
+
version_requirements: !ruby/object:Gem::Requirement
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
version: 0.1.0
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: quandl-slack
|
60
|
+
requirement: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '0'
|
65
|
+
type: :runtime
|
66
|
+
prerelease: false
|
67
|
+
version_requirements: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '0'
|
72
|
+
- !ruby/object:Gem::Dependency
|
73
|
+
name: activesupport
|
74
|
+
requirement: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 4.2.1
|
79
|
+
type: :runtime
|
80
|
+
prerelease: false
|
81
|
+
version_requirements: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 4.2.1
|
86
|
+
- !ruby/object:Gem::Dependency
|
87
|
+
name: rainbow
|
88
|
+
requirement: !ruby/object:Gem::Requirement
|
89
|
+
requirements:
|
90
|
+
- - "~>"
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: 2.0.0
|
93
|
+
type: :runtime
|
94
|
+
prerelease: false
|
95
|
+
version_requirements: !ruby/object:Gem::Requirement
|
96
|
+
requirements:
|
97
|
+
- - "~>"
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: 2.0.0
|
100
|
+
description: Help to automate opsworks project deployments with single commands.
|
101
|
+
email:
|
102
|
+
- support@quandl.com
|
103
|
+
executables:
|
104
|
+
- qops
|
105
|
+
extensions: []
|
106
|
+
extra_rdoc_files: []
|
107
|
+
files:
|
108
|
+
- README.md
|
109
|
+
- bin/qops
|
110
|
+
- lib/qops.rb
|
111
|
+
- lib/qops/cookbook/cookbook.rb
|
112
|
+
- lib/qops/deployment/app.rb
|
113
|
+
- lib/qops/deployment/helpers.rb
|
114
|
+
- lib/qops/deployment/instances.rb
|
115
|
+
- lib/qops/environment.rb
|
116
|
+
- lib/qops/helpers.rb
|
117
|
+
- lib/qops/version.rb
|
118
|
+
homepage: https://github.com/quandl/opsworks_commands
|
119
|
+
licenses:
|
120
|
+
- MIT
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: 2.2.4
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.5.1
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Helper commands for deployment of opsworks projects.
|
142
|
+
test_files: []
|
143
|
+
has_rdoc:
|