dew 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/LICENSE +22 -0
  2. data/README.md +38 -0
  3. data/Rakefile +26 -0
  4. data/bin/dew +87 -0
  5. data/config/cucumber.yaml +4 -0
  6. data/features/create-ami.feature +16 -0
  7. data/features/create-environments.feature +46 -0
  8. data/features/deploy-puge.feature +16 -0
  9. data/features/step_definitions/aws-steps.rb +101 -0
  10. data/features/step_definitions/deploy-puge-steps.rb +27 -0
  11. data/features/support/env.rb +38 -0
  12. data/features/support/hooks.rb +10 -0
  13. data/lib/dew.rb +7 -0
  14. data/lib/dew/aws_resources.yaml +122 -0
  15. data/lib/dew/base_command.rb +24 -0
  16. data/lib/dew/cloud.rb +79 -0
  17. data/lib/dew/commands.rb +6 -0
  18. data/lib/dew/commands/ami.rb +67 -0
  19. data/lib/dew/commands/console.rb +17 -0
  20. data/lib/dew/commands/console/irb_override.rb +24 -0
  21. data/lib/dew/commands/deploy.rb +114 -0
  22. data/lib/dew/commands/deploy/templates/apache.conf.erb +28 -0
  23. data/lib/dew/commands/deploy/templates/known_hosts +2 -0
  24. data/lib/dew/commands/deploy/templates/rvmrc +2 -0
  25. data/lib/dew/commands/environments.rb +110 -0
  26. data/lib/dew/commands/tidy.rb +35 -0
  27. data/lib/dew/controllers.rb +3 -0
  28. data/lib/dew/controllers/amis_controller.rb +82 -0
  29. data/lib/dew/controllers/deploy_controller.rb +10 -0
  30. data/lib/dew/controllers/environments_controller.rb +48 -0
  31. data/lib/dew/models.rb +7 -0
  32. data/lib/dew/models/account.rb +30 -0
  33. data/lib/dew/models/database.rb +32 -0
  34. data/lib/dew/models/deploy.rb +2 -0
  35. data/lib/dew/models/deploy/puge.rb +61 -0
  36. data/lib/dew/models/deploy/run.rb +19 -0
  37. data/lib/dew/models/environment.rb +199 -0
  38. data/lib/dew/models/fog_model.rb +23 -0
  39. data/lib/dew/models/profile.rb +60 -0
  40. data/lib/dew/models/server.rb +134 -0
  41. data/lib/dew/password.rb +7 -0
  42. data/lib/dew/tasks/spec.rake +14 -0
  43. data/lib/dew/validations.rb +8 -0
  44. data/lib/dew/version.rb +3 -0
  45. data/lib/dew/view.rb +39 -0
  46. data/lib/tasks/spec.rake +14 -0
  47. data/spec/dew/cloud_spec.rb +90 -0
  48. data/spec/dew/controllers/amis_controller_spec.rb +137 -0
  49. data/spec/dew/controllers/deploy_controller_spec.rb +38 -0
  50. data/spec/dew/controllers/environments_controller_spec.rb +133 -0
  51. data/spec/dew/models/account_spec.rb +47 -0
  52. data/spec/dew/models/database_spec.rb +58 -0
  53. data/spec/dew/models/deploy/puge_spec.rb +72 -0
  54. data/spec/dew/models/deploy/run_spec.rb +38 -0
  55. data/spec/dew/models/environment_spec.rb +374 -0
  56. data/spec/dew/models/fog_model_spec.rb +24 -0
  57. data/spec/dew/models/profile_spec.rb +85 -0
  58. data/spec/dew/models/server_spec.rb +190 -0
  59. data/spec/dew/password_spec.rb +11 -0
  60. data/spec/dew/spec_helper.rb +22 -0
  61. data/spec/dew/view_spec.rb +38 -0
  62. metadata +284 -0
@@ -0,0 +1,19 @@
1
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'deploy', 'puge'))
2
+
3
+ module Deploy
4
+
5
+ class Run
6
+
7
+ attr_reader :deploy_type, :environment, :opts
8
+
9
+ def initialize deploy_type, environment, opts
10
+ @deploy_type = deploy_type
11
+ @environment = environment
12
+ @opts = opts
13
+ end
14
+
15
+ def deploy
16
+ Deploy.const_get(deploy_type.capitalize).new(@environment.servers, @opts).deploy
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,199 @@
1
+ require 'dew/validations'
2
+
3
+ class Environment
4
+ include Validations
5
+
6
+ attr_reader :name, :servers, :database
7
+
8
+ def initialize name, servers=[], database=nil
9
+ @name = name
10
+ @servers = servers
11
+ @database = database
12
+ valid?
13
+ end
14
+
15
+ def valid?
16
+ Validation::validates_format_of @name, /^[a-zA-Z0-9-]+$/
17
+ end
18
+
19
+ def self.get name
20
+ servers = Server.find('Environment', name)
21
+ database = Database.get(name)
22
+ servers.length > 0 || database ? new(name, servers, database) : nil
23
+ end
24
+
25
+ def self.create(name, profile)
26
+ raise "Keypair '#{profile.keypair}' is not available in AWS #{Cloud.region}." unless Cloud.keypair_exists?(profile.keypair)
27
+ raise "AMI for '#{Cloud.region}' is not setup in the '#{profile.profile_name}' profile." unless profile.ami
28
+
29
+ environment = new(name)
30
+ Inform.info "Creating new environment %{name}", :name => name
31
+
32
+ # Creating the database is the longest running task, so do that first.
33
+ password = Password.random if profile.has_rds?
34
+ environment.add_database(profile.rds_size, password) if profile.has_rds?
35
+
36
+ (1..profile.count).each do
37
+ environment.add_server(profile.ami, profile.size, profile.keypair, profile.security_groups)
38
+ end
39
+
40
+ environment.add_elb(profile.elb_listener_ports) if profile.has_elb?
41
+
42
+ environment.wait_until_ready
43
+
44
+ environment.configure_servers_for_database password if profile.has_rds?
45
+ Inform.info "Environment %{name} ready!", :name => name
46
+ environment
47
+ end
48
+
49
+ def self.owners
50
+ Cloud.valid_servers.collect(&:tags).collect { |h| {:name => h['Environment'], :owner => h['Creator']} if h['Environment'] }.uniq.compact
51
+ end
52
+
53
+ def self.index
54
+ #environments = rows.inject({}) { |h, (name, owner)| h[name] = Environment.get(name); h }
55
+ #items = rows.collect { |name, owner| [name, owner, environments[name].servers.size, environments[name].database] }
56
+ #table(%w(name owner instance_count database), *rows).to_s.indent
57
+ Inform.info View.new('Environments', owners, %w(name owner)).index
58
+ end
59
+
60
+ def show
61
+ result = ""
62
+
63
+ result << "ENVIRONMENT:\n"
64
+
65
+ result << "NAME: #{name}\n"
66
+ result << "REGION: #{Cloud.region}\n"
67
+ result << "ACCOUNT: #{Cloud.account.aws_user_id}\n"
68
+
69
+ unless servers.empty?
70
+ #keys = %w(id flavor_id public_ip_address state created_at key_name tags)
71
+ keys = %w(id flavor_id public_ip_address state created_at groups key_name availability_zone creator)
72
+ servers_view = View.new('SERVERS', servers, keys)
73
+ result << servers_view.index
74
+ end
75
+
76
+ if elb
77
+ elbs_view = View.new('ELB', [elb], %w(created_at dns_name instances availability_zones))
78
+ result << elbs_view.show(0)
79
+ end
80
+
81
+ if database
82
+ databases_view = View.new('DATABASE', [database], %w(flavor_id state created_at availability_zone db_security_groups))
83
+ result << databases_view.show(0)
84
+ end
85
+
86
+ Inform.info result
87
+ end
88
+
89
+ def destroy
90
+ servers.each do |server|
91
+ Inform.info("Destroying server %{serverid}", :serverid => server.id) do
92
+ server.destroy
93
+ end
94
+ end
95
+ if database
96
+ Inform.info("Destroying database") do
97
+ database.destroy(nil)
98
+ end
99
+ end
100
+ if has_elb?
101
+ Inform.info("Destroying load balancer") do
102
+ Cloud.elb.delete_load_balancer(name)
103
+ end
104
+ end
105
+ end
106
+
107
+ def add_server ami, size, keypair, groups
108
+ Inform.info "Adding server using AMI %{ami} of size %{size}, keypair %{keypair} and security groups %{groups}",
109
+ :ami => ami, :size => size, :keypair => keypair, :groups => groups do
110
+ server = Server.create!( ami, size, keypair, groups )
111
+ server.add_tag('Environment', name)
112
+ server.add_tag('Creator', ENV['USER'])
113
+ server_name = "#{name} #{servers.count + 1}"
114
+ server.add_tag('Name', server_name)
115
+ Inform.debug("%{name} ID: %{id} AZ: %{az}", :name => server_name, :id =>server.id, :az => server.availability_zone)
116
+ servers << server
117
+ end
118
+ end
119
+
120
+ def add_server_to_elb server
121
+ Inform.debug("Adding %{id} to ELB", :id => server.id)
122
+ Cloud.elb.register_instances_with_load_balancer([server.id], name)
123
+ end
124
+
125
+ def remove_server_from_elb server
126
+ Inform.debug("Removing %{id} from ELB", :id => server.id)
127
+ Cloud.elb.deregister_instances_from_load_balancer([server.id], name)
128
+ end
129
+
130
+ def add_elb listener_ports=[]
131
+ zones = servers.map {|s| s.availability_zone}.uniq
132
+ Inform.info "Adding Load Balancer for availability zone(s) %{zones}", :zones => zones.join(', ') do
133
+ Cloud.elb.create_load_balancer(zones, name, listeners_from_ports(listener_ports))
134
+ Inform.debug("ELB created, adding instances...")
135
+ Cloud.elb.register_instances_with_load_balancer(servers.map {|s| s.id}, name)
136
+ end
137
+ end
138
+
139
+ def add_database size, password
140
+ Inform.info "Adding Database %{name} of size %{size} with master password %{password}",
141
+ :name => name, :size => size, :password => password do
142
+ @database = Database.create!(name, size, password)
143
+ end
144
+ end
145
+
146
+ def wait_until_ready
147
+ Inform.info "Waiting for servers to become ready" do
148
+ servers.each do |server|
149
+ ip_permissions = server.groups.map { |group_name| Cloud.security_groups[group_name] }.compact.collect(&:ip_permissions)
150
+ if ip_permissions.flatten.empty?
151
+ Inform.warning "Server %{id} has no ip_permissions in its security groups %{security_group_names}", :id => server.id, :security_group_names => server.groups.inspect
152
+ else
153
+ Inform.debug "Trying to connect to %{id} using ip_permissions %{ip_permissions}", :id => server.id, :ip_permissions => ip_permissions
154
+ server.wait_until_ready
155
+ end
156
+ end
157
+ end
158
+ if database
159
+ Inform.info "Waiting for database to become ready" do
160
+ database.wait_until_ready if database
161
+ end
162
+ end
163
+ end
164
+
165
+ def configure_servers_for_database password
166
+ Inform.info "Configuring servers for database" do
167
+ servers.each do |server|
168
+ Inform.debug "%{id}", :id => server.id
169
+ server.configure_for_database(database, password)
170
+ end
171
+ end
172
+ end
173
+
174
+ def elb
175
+ @elb ||= Cloud.elb.load_balancers.detect { |elb| elb.id == name } # XXX - need to refactor
176
+ end
177
+
178
+ def has_elb?
179
+ elb_descriptions.select { |elb|
180
+ elb['LoadBalancerName'] == name
181
+ }.length > 0
182
+ end
183
+
184
+ private
185
+
186
+ def listeners_from_ports ports
187
+ ports.map do |port|
188
+ {
189
+ 'Protocol' => 'TCP',
190
+ 'LoadBalancerPort' => port,
191
+ 'InstancePort' => port,
192
+ }
193
+ end
194
+ end
195
+
196
+ def elb_descriptions name = nil
197
+ Cloud.elb.describe_load_balancers(name).body['DescribeLoadBalancersResult']['LoadBalancerDescriptions']
198
+ end
199
+ end
@@ -0,0 +1,23 @@
1
+ class FogModel
2
+ def initialize fog_object
3
+ @fog_object = fog_object
4
+ end
5
+
6
+ def id
7
+ @fog_object.id
8
+ end
9
+
10
+ def wait_until_ready
11
+ @fog_object.wait_for { ready? }
12
+ end
13
+
14
+ def method_missing method_sym, *args, &block
15
+ @fog_object.send(method_sym, *args, &block)
16
+ end
17
+
18
+ protected
19
+
20
+ def fog_object
21
+ @fog_object
22
+ end
23
+ end
@@ -0,0 +1,60 @@
1
+ require 'yaml'
2
+
3
+ class Profile
4
+
5
+ attr_reader :profile_name
6
+ attr_accessor :ami, :size, :security_groups, :keypair, :count
7
+ attr_accessor :rds_size, :elb_listener_ports
8
+
9
+ AWS_RESOURCES = YAML.load(File.read(File.join(File.dirname(__FILE__), '..', 'aws_resources.yaml')))
10
+
11
+ def self.read(profile_name)
12
+ file = File.read(File.join(ENV['HOME'], '.dew', 'profiles', "#{profile_name}.yaml"))
13
+ yaml = YAML.load(file)
14
+ profile = new(profile_name)
15
+ if yaml['instances']
16
+ profile.ami = yaml['instances']['amis'][Cloud.region]
17
+ profile.size = yaml['instances']['size']
18
+ profile.security_groups = yaml['instances']['security-groups'] || ['default']
19
+ profile.keypair = yaml['instances']['keypair']
20
+ profile.count = yaml['instances']['count'].to_i
21
+ end
22
+ if yaml['elb']
23
+ profile.elb_listener_ports = yaml['elb']['listener_ports']
24
+ end
25
+ if yaml['rds']
26
+ profile.rds_size = yaml['rds']['size']
27
+ end
28
+ profile
29
+ end
30
+
31
+ def has_elb?
32
+ elb_listener_ports != nil
33
+ end
34
+
35
+ def has_rds?
36
+ rds_size != nil
37
+ end
38
+
39
+ def initialize(profile_name)
40
+ @profile_name = profile_name
41
+ # :ami, :size, :security_groups, :keypair, :count
42
+ # :rds_size, :elb_listener_ports
43
+ end
44
+
45
+ def to_s
46
+ db_instance_str = "%{memory} memory, %{processor} processor, %{platform} platform, %{io_performance} I/O performance"
47
+ #instance_str = "%{memory} memory, %{processor} processor, %{storage} storage, %{platform} platform, %{io_performance} I/O performance"
48
+ instance_str = "%{memory} GB memory, %{processor} ECUs processor, %{storage} GB storage, %{platform}-bit platform, %{io_performance} I/O performance"
49
+ flavor = Cloud.compute.flavors.detect { |f| f.id == size }
50
+ instance_hash = { :memory => flavor.ram.to_s, :processor => flavor.cores.to_s, :storage => flavor.disk.to_s, :platform => flavor.bits.to_s, :io_performance => '??' }
51
+ table { |t|
52
+ t << [ "#{count} instance#{'s' if count > 1}", "#{size.inspect} (#{instance_str % instance_hash })"]
53
+ t << ['disk image', ami.inspect]
54
+ t << ['load balancer', "listener ports: #{elb_listener_ports.inspect}"] if has_elb?
55
+ t << ['database', "#{rds_size.inspect} (#{db_instance_str % AWS_RESOURCES['db_instance_types'][rds_size]})"] if has_rds?
56
+ t << ['security groups', security_groups.inspect]
57
+ t << ['keypair', keypair.inspect]
58
+ }.to_s
59
+ end
60
+ end
@@ -0,0 +1,134 @@
1
+ require 'gofer'
2
+
3
+ class Server < FogModel
4
+ TEN_SECONDS = 10
5
+ SIXTY_SECONDS = 60
6
+ TWO_MINUTES = 120
7
+ RUNNING_SERVER_STATES = %w{pending running}
8
+
9
+ def self.create! ami, size, keypair, groups
10
+ new(Cloud.compute.servers.create(:image_id => ami, :flavor_id => size, :key_name => keypair, :groups => groups))
11
+ end
12
+
13
+ def self.find tag_name, tag_value
14
+ Cloud.compute.servers.all("tag:#{tag_name}" => tag_value).select{|s| RUNNING_SERVER_STATES.include?(s.state)}.map {|s| new s }
15
+ end
16
+
17
+ def creator
18
+ @creator ||= fog_object.tags["Creator"]
19
+ end
20
+
21
+ def add_tag key, val
22
+ try_for(SIXTY_SECONDS) {
23
+ Cloud.compute.tags.create(:resource_id => id, :key => key, :value => val)
24
+ }
25
+ end
26
+
27
+ def configure_for_database database, password
28
+ ssh.write(database.db_environment_file(password), '/tmp/envfile')
29
+ ssh.run('sudo mv /tmp/envfile /etc/environment')
30
+ end
31
+
32
+ def username
33
+ 'ubuntu'
34
+ end
35
+
36
+ def ssh
37
+ Gofer::Host.new(public_ip_address, username, :key_data => [File.read(Cloud.keyfile_path(key_name))], :paranoid => false, :quiet => true)
38
+ end
39
+
40
+ def wait_until_ready ssh_timeout=TWO_MINUTES
41
+ super()
42
+ Inform.debug("%{id} online at %{ip}, waiting for SSH connection...", :id => id, :ip => public_ip_address)
43
+ wait_for_ssh ssh_timeout
44
+ Inform.debug("Connected to %{id} via SSH successfully", :id => id)
45
+ end
46
+
47
+ def create_ami ami_name
48
+ image_id = Cloud.compute.create_image(id, ami_name, "Created by #{ENV['USER']} on #{Time.now.strftime("%Y-%m-%d")}").body['imageId']
49
+
50
+ Inform.debug("Created image at %{id}, waiting for AWS to recognize it...", :id => image_id)
51
+ # Sometimes takes a while for AWS to realise there's a new image...
52
+ image = Timeout::timeout(SIXTY_SECONDS) do
53
+ image = nil
54
+ while image == nil
55
+ image = Cloud.compute.images.get(image_id)
56
+ end
57
+ image
58
+ end
59
+ Inform.debug("Image recognized, waiting for it to become available...")
60
+
61
+ image.wait_for { state == 'available' }
62
+ Inform.debug("Image available, sharing with other accounts...")
63
+
64
+ Account.user_ids.each do |user_id|
65
+ Inform.debug("Sharing %{id} with %{user_id}", :id => image_id, :user_id => user_id)
66
+ Cloud.compute.modify_image_attributes(image_id, 'launchPermission', 'add', 'UserId' => user_id)
67
+ end
68
+ image_id
69
+ end
70
+
71
+ def credentials
72
+ @credentials ||= if key_name
73
+ keyfile_path = Cloud.keyfile_path(key_name)
74
+
75
+ sanitize_key_file(key_name, keyfile_path)
76
+
77
+ "-i #{keyfile_path} -o StrictHostKeyChecking=no #{username}@#{public_ip_address}"
78
+ else
79
+ Inform.warning("Server %{id} has no key and therefore can not be accessed.", :id => id)
80
+ false
81
+ end
82
+ end
83
+
84
+ private
85
+
86
+ def sanitize_key_file name, path
87
+ begin
88
+ stat = File.stat(path)
89
+ raise "Keyfile at #{keyfile_path} not owned by #{ENV['USER']}, can't SSH" if stat.uid != Process.euid
90
+ if (stat.mode & 077) != 0
91
+ Inform.info("Changing permissions on key at %{path} to 0600", :path => path) do
92
+ File.chmod(0600, path)
93
+ end
94
+ end
95
+ rescue Errno::ENOENT
96
+ raise "Can't find keyfile for #{name} under this account/region (looking in #{path})"
97
+ end
98
+ end
99
+
100
+ def wait_for_ssh timeout
101
+ try_for(timeout) {
102
+ begin
103
+ Timeout::timeout(15) do
104
+ ssh.run('uptime')
105
+ end
106
+ # SshTimeout magic is here due to a weird bug with Ruby 1.8.7 where the Timeout::Error
107
+ # will not be caught by wait_for_proc!
108
+ rescue Timeout::Error => e
109
+ raise "SSH Timeout Encountered: #{e.to_s}"
110
+ end
111
+ }
112
+ end
113
+
114
+ def try_for(timeout, &block)
115
+ start_time = Time.now
116
+ time_spent = 0
117
+ success = false
118
+ last_exception = nil
119
+ while !success && time_spent < timeout
120
+ begin
121
+ block.call
122
+ success = true
123
+ rescue => e
124
+ time_spent = Time.now - start_time
125
+ Inform.debug("Exception: %{m} (%{c}), in block %{block} after %{time_spent}s (retrying until %{timeout}s)...", :c => e.class.to_s, :m => e.message, :block => block, :time_spent => time_spent.to_i, :timeout => timeout)
126
+ sleep 1
127
+ last_exception = e
128
+ end
129
+ end
130
+ #raise unless success
131
+ raise last_exception unless success
132
+ end
133
+
134
+ end
@@ -0,0 +1,7 @@
1
+ class Password
2
+
3
+ def self.random length=12
4
+ (0...length).map{ ('a'..'z').to_a[rand(26)] }.join
5
+ end
6
+
7
+ end