dew 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/LICENSE +22 -0
  2. data/README.md +38 -0
  3. data/Rakefile +26 -0
  4. data/bin/dew +87 -0
  5. data/config/cucumber.yaml +4 -0
  6. data/features/create-ami.feature +16 -0
  7. data/features/create-environments.feature +46 -0
  8. data/features/deploy-puge.feature +16 -0
  9. data/features/step_definitions/aws-steps.rb +101 -0
  10. data/features/step_definitions/deploy-puge-steps.rb +27 -0
  11. data/features/support/env.rb +38 -0
  12. data/features/support/hooks.rb +10 -0
  13. data/lib/dew.rb +7 -0
  14. data/lib/dew/aws_resources.yaml +122 -0
  15. data/lib/dew/base_command.rb +24 -0
  16. data/lib/dew/cloud.rb +79 -0
  17. data/lib/dew/commands.rb +6 -0
  18. data/lib/dew/commands/ami.rb +67 -0
  19. data/lib/dew/commands/console.rb +17 -0
  20. data/lib/dew/commands/console/irb_override.rb +24 -0
  21. data/lib/dew/commands/deploy.rb +114 -0
  22. data/lib/dew/commands/deploy/templates/apache.conf.erb +28 -0
  23. data/lib/dew/commands/deploy/templates/known_hosts +2 -0
  24. data/lib/dew/commands/deploy/templates/rvmrc +2 -0
  25. data/lib/dew/commands/environments.rb +110 -0
  26. data/lib/dew/commands/tidy.rb +35 -0
  27. data/lib/dew/controllers.rb +3 -0
  28. data/lib/dew/controllers/amis_controller.rb +82 -0
  29. data/lib/dew/controllers/deploy_controller.rb +10 -0
  30. data/lib/dew/controllers/environments_controller.rb +48 -0
  31. data/lib/dew/models.rb +7 -0
  32. data/lib/dew/models/account.rb +30 -0
  33. data/lib/dew/models/database.rb +32 -0
  34. data/lib/dew/models/deploy.rb +2 -0
  35. data/lib/dew/models/deploy/puge.rb +61 -0
  36. data/lib/dew/models/deploy/run.rb +19 -0
  37. data/lib/dew/models/environment.rb +199 -0
  38. data/lib/dew/models/fog_model.rb +23 -0
  39. data/lib/dew/models/profile.rb +60 -0
  40. data/lib/dew/models/server.rb +134 -0
  41. data/lib/dew/password.rb +7 -0
  42. data/lib/dew/tasks/spec.rake +14 -0
  43. data/lib/dew/validations.rb +8 -0
  44. data/lib/dew/version.rb +3 -0
  45. data/lib/dew/view.rb +39 -0
  46. data/lib/tasks/spec.rake +14 -0
  47. data/spec/dew/cloud_spec.rb +90 -0
  48. data/spec/dew/controllers/amis_controller_spec.rb +137 -0
  49. data/spec/dew/controllers/deploy_controller_spec.rb +38 -0
  50. data/spec/dew/controllers/environments_controller_spec.rb +133 -0
  51. data/spec/dew/models/account_spec.rb +47 -0
  52. data/spec/dew/models/database_spec.rb +58 -0
  53. data/spec/dew/models/deploy/puge_spec.rb +72 -0
  54. data/spec/dew/models/deploy/run_spec.rb +38 -0
  55. data/spec/dew/models/environment_spec.rb +374 -0
  56. data/spec/dew/models/fog_model_spec.rb +24 -0
  57. data/spec/dew/models/profile_spec.rb +85 -0
  58. data/spec/dew/models/server_spec.rb +190 -0
  59. data/spec/dew/password_spec.rb +11 -0
  60. data/spec/dew/spec_helper.rb +22 -0
  61. data/spec/dew/view_spec.rb +38 -0
  62. metadata +284 -0
@@ -0,0 +1,28 @@
1
+ <VirtualHost *:80>
2
+ ServerAdmin admin@playup.com
3
+
4
+ PassEnv PUGE_DB_HOST PUGE_DB_NAME PUGE_DB_USERNAME PUGE_DB_PASSWORD
5
+
6
+ PassengerFriendlyErrorPages off
7
+ PassengerMaxRequests 50
8
+
9
+ RailsEnv <%= rails_env %>
10
+
11
+ DocumentRoot <%= working_directory %>/public
12
+ <Directory <%= working_directory %>/public>
13
+ allow from all
14
+ Options -MultiViews
15
+ AllowOverride None
16
+ Order allow,deny
17
+ </Directory>
18
+
19
+ ErrorLog /var/log/apache2/<%= application_name %>-error.log
20
+
21
+ # Possible values include: debug, info, notice, warn, error, crit,
22
+ # alert, emerg.
23
+ LogLevel warn
24
+
25
+ CustomLog /var/log/apache2/<%= application_name %>-access.log combined
26
+
27
+ PassengerPreStart http://localhost/status
28
+ </VirtualHost>
@@ -0,0 +1,2 @@
1
+ |1|cFg9wYM1j4pHeyeYQaENVE2Jb/g=|hA1okVZROh9DyjcslQNVy2FAH7Q= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
2
+ |1|WQr8sEZnbYuTqpjNc5JTjJUZLuc=|PDsO3bSoH/tWy/fpJ4+GUluZC+I= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
@@ -0,0 +1,2 @@
1
+ # rvm_trust_rvmrcs_flag=1
2
+ # unused
@@ -0,0 +1,110 @@
1
+ require 'dew/controllers/environments_controller'
2
+
3
+
4
+ class EnvironmentsCommand < Clamp::Command
5
+
6
+ #STATUS_KEYS=%w(arch cpu_count cpu_freq disk ec2_cost ip_address load_average mem_available mem_used network processes reboot_required release time_utc updates_available uptime users)
7
+ STATUS_KEYS=%w(ip_address arch cpu_count cpu_freq disk ec2_cost load_average mem_available mem_used network processes release time_utc uptime)
8
+ STATUS_CMD="'[ -d ~/.byobu ] || mkdir ~/.byobu; touch ~/.byobu/status; for cmd in #{STATUS_KEYS.join(' ')}; do echo `/usr/lib/byobu/$cmd 2>/dev/null`; done'"
9
+
10
+ def controller
11
+ @controller ||= EnvironmentsController.new
12
+ end
13
+
14
+ default_subcommand "index", "Show environments" do
15
+
16
+ def execute
17
+ controller.index
18
+ end
19
+
20
+ end
21
+
22
+ subcommand "create", "Create a new environment" do
23
+
24
+ option ['-f', '--force'], :flag, "Don't ask for confirmation before creating", :default => false
25
+ parameter "PROFILE", "Profile describing resources to be created", :attribute_name => 'profile_name'
26
+ parameter "ENVIRONMENT_NAME", "Name of the environment"
27
+
28
+ def execute
29
+ controller.create(environment_name, profile_name, :force => force?)
30
+ end
31
+
32
+ end
33
+
34
+ subcommand "show", "Show environment" do
35
+
36
+ parameter "ENVIRONMENT_NAME", "Name of the environment"
37
+
38
+ def execute
39
+ controller.show(environment_name)
40
+ end
41
+
42
+ end
43
+
44
+ subcommand "status", "Get the status of an instance (NB. requires byobu to be installed)" do
45
+ parameter "[ENVIRONMENT_NAME]", "Name of the environment"
46
+ option ['-i', '--instance'], 'INSTANCE_NUMBER', "Which instance to SSH to", :default => 1 do |s|
47
+ Integer(s)
48
+ end
49
+
50
+ def execute
51
+ environment_owners = Environment.owners
52
+ environment_owners = [environment_owners.detect { |o| o[:name] == environment_name }] if environment_name
53
+ rows = []
54
+ environment_owners.collect { |o|
55
+ environment_name = o[:name]
56
+ environment_owner = o[:owner]
57
+ instance_count = Environment.get(environment_name).servers.size
58
+ (1..instance_count).collect { |instance_no|
59
+ server = get_server(environment_name, instance_no)
60
+ if server.credentials
61
+ command = "ssh #{server.credentials} #{STATUS_CMD}"
62
+ Inform.debug("Running %{command}", :command => command)
63
+ rows << [environment_name, environment_owner]+`#{command}`.split("\n")
64
+ end
65
+ }
66
+ }
67
+ Inform.info "\n#{rows.empty? ? "None" : table(%w(env owner)+STATUS_KEYS, *rows)}"
68
+ end
69
+ end
70
+
71
+ subcommand "ssh", "SSH to an environment" do
72
+ parameter "ENVIRONMENT_NAME", "Name of the environment"
73
+ option ['-i', '--instance'], 'INSTANCE_NUMBER', "Which instance to SSH to", :default => 1 do |s|
74
+ Integer(s)
75
+ end
76
+ option ['-p', '--print'], :flag, "Print the SSH credentials instead of actually performing the SSH operation", :default => false
77
+
78
+ def execute
79
+ server = get_server(environment_name, instance)
80
+ if server.credentials
81
+ if print?
82
+ puts server.credentials
83
+ else
84
+ command = "ssh #{server.credentials}"
85
+ Inform.debug("Running %{command}", :command => command)
86
+ system command
87
+ end
88
+ end
89
+ end
90
+ end
91
+
92
+ subcommand "destroy", "Destroy an existing environment" do
93
+
94
+ option ['-f', '--force'], :flag, "Don't ask for confirmation before destroying", :default => false
95
+ parameter "ENVIRONMENT_NAME", "Name of the environment to be destroyed"
96
+
97
+ def execute
98
+ controller.destroy(environment_name, :force => force?)
99
+ end
100
+
101
+ end
102
+
103
+ def get_server(environment_name, instance_no)
104
+ env = Environment.get(environment_name)
105
+ server = env.servers[instance_no - 1]
106
+ raise "Environment only has #{env.servers.length} instances, can't SSH to instance ##{instance}" unless server
107
+ server
108
+ end
109
+
110
+ end
@@ -0,0 +1,35 @@
1
+ class TidyCommand < Clamp::Command
2
+
3
+ option ['--[no-]clean-environments'], :flag, "Clean up environments", :default => true
4
+ option ['--[no-]clean-amis'], :flag, "Clean up AMIs", :default => true
5
+ option ['--noop'], :flag, "Print out what we'd do instead of doing it", :default => false
6
+
7
+ def tidy_environments
8
+ Inform.info("Tidying up Environments...")
9
+ names = Cloud.valid_servers.collect(&:tags).collect { |h| h['Environment'] if h['Environment'] }
10
+ names << Cloud.rds.servers.all.select {|a| a.state == 'available'}.map(&:id)
11
+ names = names.flatten.uniq
12
+ names = names.grep /^cuke-/
13
+ names.each do |name|
14
+ Inform.info("Destroying environment #{name}")
15
+ Environment.get(name).destroy unless noop?
16
+ end
17
+ end
18
+
19
+ def tidy_amis
20
+ Inform.info("Tidying up AMIS...")
21
+ amis = Cloud.compute.images.all
22
+ amis = amis.select {|a| a.name =~ /^cuke-/}
23
+ amis.each do |ami|
24
+ Inform.info("Destroying ami #{ami.name}") do
25
+ ami.deregister unless noop?
26
+ end
27
+ end
28
+ end
29
+
30
+ def execute
31
+ Inform.warning("--noop passed, no changes will be made!") if noop?
32
+ tidy_environments if clean_environments?
33
+ tidy_amis if clean_amis?
34
+ end
35
+ end
@@ -0,0 +1,3 @@
1
+ require 'dew/controllers/amis_controller'
2
+ require 'dew/controllers/deploy_controller'
3
+ require 'dew/controllers/environments_controller'
@@ -0,0 +1,82 @@
1
+ require 'dew/controllers/environments_controller'
2
+
3
+ class AMIsController
4
+
5
+ AMI_PROFILE = 'ami-prototype'
6
+
7
+ def create ami_name, puppet_node_name
8
+ Inform.info("Creating new AMI %{ami_name} using puppet node %{puppet}", :ami_name => ami_name, :puppet => puppet_node_name)
9
+ environment_name = ami_name + '-prototype-' + $$.to_s
10
+
11
+ environment = Environment.create(environment_name, Profile.read(AMI_PROFILE))
12
+ @prototype = environment.servers.first
13
+ Inform.debug("Using server %{id} at %{ip} as our prototype.", :id => @prototype.id, :ip => @prototype.public_ip_address)
14
+
15
+ Inform.debug("Installing puppet...")
16
+ install_puppet_on_prototype
17
+ Inform.debug("Copying puppet configuration... ")
18
+ copy_puppet_to_prototype
19
+ Inform.debug("Running puppet node %{node}... ", :node => puppet_node_name)
20
+ run_puppet_node_on_prototype puppet_node_name
21
+
22
+ ami_id = Inform.info "Creating new ami with name %{ami_name}", :ami_name => ami_name do
23
+ @prototype.create_ami ami_name
24
+ end
25
+ Inform.info("New AMI id is %{ami_id}", :ami_id => ami_id)
26
+ environment.destroy
27
+ end
28
+
29
+ def index
30
+ # Inform.info("AMIs:\n#{Cloud.compute.images.all('owner_id' => Cloud.account.aws_user_id)}")
31
+ # /home/chris/.rvm/gems/ruby-1.9.2-p180@AWS/gems/excon-0.6.3/lib/excon/connection.rb:179:in `request': InvalidParameterValue => The filter 'owner_id' is invalid (Fog::Service::Error)
32
+ my_amis = Cloud.compute.images.all.select { |x| x.owner_id == Cloud.account.aws_user_id }
33
+ keys = %w(name id state architecture kernel_id description)
34
+ Inform.info(View.new('My AMIs', my_amis, keys).index)
35
+ end
36
+
37
+ def show ami_name
38
+ my_amis = Cloud.compute.images.all('name' => ami_name)
39
+ raise "AMI named #{ami_name} not found!" if my_amis.empty?
40
+ keys = %w(id architecture block_device_mapping description location owner_id state type is_public kernel_id platform product_codes ramdisk_id root_device_type root_device_name tags name)
41
+ Inform.info(View.new('My AMIs', my_amis, keys).show(0))
42
+ end
43
+
44
+ def destroy ami_name, opts={}
45
+ ami = Cloud.compute.images.all('name' => ami_name).first
46
+ raise "AMI named #{ami_name} not found!" unless ami
47
+ if opts[:force] || agree("<%= color('Are you sure?', YELLOW, BOLD) %> ")
48
+ Inform.info("Destroying AMI named %{n}", :n => ami_name) do
49
+ ami.deregister
50
+ end
51
+ else
52
+ Inform.info "Aborting AMI destruction"
53
+ end
54
+ end
55
+
56
+ private
57
+
58
+ def ssh
59
+ @ssh ||= @prototype.ssh
60
+ end
61
+
62
+ def install_puppet_on_prototype
63
+ Inform.info("Installing puppet") do
64
+ ssh.run('sudo apt-get update', :quiet_stderr => true)
65
+ ssh.run('sudo apt-get -q -y install puppet', :quiet_stderr => true) # chatty
66
+ end
67
+ end
68
+
69
+ def copy_puppet_to_prototype
70
+ Inform.info("Uploading puppet configuration") do
71
+ ssh.upload(File.join(ENV['HOME'], '.dew', 'puppet'), '/tmp/puppet')
72
+ ssh.run("sudo rm -rf /etc/puppet")
73
+ ssh.run("sudo mv /tmp/puppet /etc/puppet")
74
+ end
75
+ end
76
+
77
+ def run_puppet_node_on_prototype puppet_node_name
78
+ Inform.info("Running puppet node %{name} (this may take a while)", :name => puppet_node_name) do
79
+ ssh.run("sudo puppet /etc/puppet/manifests/nodes/#{puppet_node_name}.pp #{puppet_node_name} > /tmp/puppet_run_log 2>&1")
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,10 @@
1
+ class DeployController
2
+
3
+ def create deploy_type, environment_name, opts
4
+ if Environment.get(environment_name).servers.empty?
5
+ raise "Environment #{environment_name.inspect} doesn't exist or appears to have all instances already terminated"
6
+ end
7
+
8
+ Deploy::Run.new(deploy_type, Environment.get(environment_name), opts).deploy
9
+ end
10
+ end
@@ -0,0 +1,48 @@
1
+ require 'dew/models/profile'
2
+
3
+ class EnvironmentsController
4
+
5
+ def create(name, profile_name, opts={})
6
+ profile = Profile.read(profile_name)
7
+ if opts[:force] || (
8
+ Inform.info("About to create environment %{name} using the following profile:\n%{profile}" , :name => name, :profile => profile.to_s)
9
+ agree("<%= color('Do you wish to continue?', YELLOW, BOLD) %> ")
10
+ )
11
+ environment = Environment.create(name, profile)
12
+ environment.show
13
+ environment
14
+ else
15
+ Inform.info "Aborting environment creation"
16
+ end
17
+ end
18
+
19
+ def index
20
+ Environment.index
21
+ end
22
+
23
+ def show(name)
24
+ before_get_environment name
25
+
26
+ @environment.show
27
+ end
28
+
29
+ def destroy name, opts={}
30
+ before_get_environment name
31
+
32
+ @environment.show
33
+ Inform.info "Destroying environment %{name} ...", :name => name
34
+ if opts[:force] || agree("<%= color('Are you sure?', YELLOW, BOLD) %> ")
35
+ @environment.destroy
36
+ Inform.info "Environment %{name} destroyed", :name => name
37
+ else
38
+ Inform.info "Aborting environment destruction"
39
+ end
40
+ end
41
+
42
+ # a rough before filter
43
+ def before_get_environment(name)
44
+ @environment = Environment.get(name)
45
+ raise "Environment named #{name} not found!" unless @environment
46
+ end
47
+
48
+ end
@@ -0,0 +1,7 @@
1
+ require 'fog'
2
+ require 'dew/models/account'
3
+ require 'dew/models/fog_model'
4
+ require 'dew/models/deploy'
5
+ require 'dew/models/server'
6
+ require 'dew/models/database'
7
+ require 'dew/models/environment'
@@ -0,0 +1,30 @@
1
+ require 'yaml'
2
+
3
+ class Account
4
+
5
+ def self.read(account_name)
6
+ Account.new YAML.load File.read File.join [ ENV['HOME'], '.dew', 'accounts', "#{account_name}.yaml" ]
7
+ end
8
+
9
+ def self.user_ids
10
+ Dir[File.join(ENV['HOME'], '.dew', 'accounts', '*.yaml')].map do |filename|
11
+ Account.read(File.basename(filename).gsub(/.yaml$/, '')).aws_user_id
12
+ end
13
+ end
14
+
15
+ def aws_access_key_id
16
+ @yaml['aws']['access_key_id']
17
+ end
18
+
19
+ def aws_secret_access_key
20
+ @yaml['aws']['secret_access_key']
21
+ end
22
+
23
+ def aws_user_id
24
+ @yaml['aws']['user_id'].gsub('-', '')
25
+ end
26
+
27
+ def initialize(yaml)
28
+ @yaml = yaml
29
+ end
30
+ end
@@ -0,0 +1,32 @@
1
+ class Database < FogModel
2
+
3
+ def self.create! name, size, password
4
+ new Cloud.rds.servers.create(
5
+ :engine => 'MySQL',
6
+ :master_username => 'root',
7
+ :password => password,
8
+ :id => name,
9
+ :allocated_storage => '5',
10
+ :flavor_id => size
11
+ )
12
+ end
13
+
14
+ def self.get name
15
+ db = Cloud.rds.servers.get(name)
16
+ new db if db
17
+ end
18
+
19
+ def public_address
20
+ fog_object.endpoint['Address']
21
+ end
22
+
23
+ def db_environment_file password
24
+ <<-EOF
25
+ PUGE_DB_HOST=#{fog_object.endpoint['Address']}
26
+ PUGE_DB_NAME=#{id}
27
+ PUGE_DB_USERNAME=#{master_username}
28
+ PUGE_DB_PASSWORD=#{password}
29
+ EOF
30
+ end
31
+
32
+ end
@@ -0,0 +1,2 @@
1
+ require 'dew/models/deploy/puge'
2
+ require 'dew/models/deploy/run'
@@ -0,0 +1,61 @@
1
+ module Deploy
2
+
3
+ class Puge
4
+
5
+ def initialize servers, opts
6
+ @servers = servers
7
+ @opts = opts
8
+ end
9
+
10
+ def deploy
11
+ execute_in_parallel_and_wait Proc.new { |server|
12
+ Inform.info("%{server_id}: Cloning PUGE and checking out tag %{tag}", :server_id => server.id, :tag => @opts['tag'])
13
+ upload_and_run(server, 'clone_puge.sh', @opts['tag'])
14
+
15
+ Inform.info("%{server_id}: Running bundle install", :server_id => server.id)
16
+ upload_and_run(server, 'bundle_install.sh')
17
+ }
18
+
19
+ # This task cannot run in parallel as there's only one RDS
20
+ #
21
+ Inform.info("%{server_id}: Setting up Rails database using %{rails_env} Rails environment", :server_id => @servers.first.id, :rails_env => @opts['rails_env'])
22
+ upload_and_run(@servers.first, 'setup_rails_database.sh', @opts['rails_env'])
23
+
24
+ execute_in_parallel_and_wait Proc.new { |server|
25
+ Inform.info("%{server_id}: Generating PUGE WAR", :server_id => server.id)
26
+ upload_and_run(server, 'generate_puge_war.sh', @opts['rails_env'])
27
+
28
+ Inform.info("%{server_id}: Copying PUGE WAR into Tomcat directory", :server_id => server.id)
29
+ upload_and_run(server, 'copy_puge_war_into_tomcat.sh')
30
+
31
+ Inform.info("%{server_id}: Restarting Tomcat", :server_id => server.id)
32
+ upload_and_run(server, 'restart_tomcat.sh')
33
+ }
34
+ end
35
+
36
+ private
37
+
38
+ def execute_in_parallel_and_wait proc
39
+ threads = []
40
+
41
+ @servers.each do |server|
42
+ threads << Thread.new { proc.call(server) }
43
+ end
44
+
45
+ # wait for all threads to finish
46
+ #
47
+ threads.each do |thread|
48
+ thread.join
49
+ end
50
+ end
51
+
52
+ def upload_script server, script
53
+ server.ssh.upload(File.join(ENV['HOME'], '.dew', 'deploy', 'puge', script), '.')
54
+ end
55
+
56
+ def upload_and_run server, script, *args
57
+ upload_script(server, script)
58
+ server.ssh.run(['./' + script, args.map {|a| "'#{a}'"}].flatten.join(" "))
59
+ end
60
+ end
61
+ end