ScholarNexus-ec2onrails 0.9.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. data/CHANGELOG +180 -0
  2. data/COPYING +339 -0
  3. data/Manifest +167 -0
  4. data/README.textile +211 -0
  5. data/Rakefile +37 -0
  6. data/TODO +104 -0
  7. data/ec2onrails.gemspec +42 -0
  8. data/examples/Capfile +3 -0
  9. data/examples/deploy.rb +107 -0
  10. data/examples/s3.yml +9 -0
  11. data/lib/ec2onrails/capistrano_utils.rb +43 -0
  12. data/lib/ec2onrails/recipes/db.rb +359 -0
  13. data/lib/ec2onrails/recipes/deploy.rb +30 -0
  14. data/lib/ec2onrails/recipes/server.rb +489 -0
  15. data/lib/ec2onrails/recipes.rb +141 -0
  16. data/lib/ec2onrails/version.rb +31 -0
  17. data/lib/ec2onrails.rb +20 -0
  18. data/server/build-ec2onrails.sh +44 -0
  19. data/server/files/etc/aliases +5 -0
  20. data/server/files/etc/aliases.db +0 -0
  21. data/server/files/etc/apache2/apache2.conf +295 -0
  22. data/server/files/etc/apache2/conf.d/app.proxy_cluster.conf +7 -0
  23. data/server/files/etc/apache2/conf.d/app.proxy_frontend.conf +10 -0
  24. data/server/files/etc/apache2/mods-available/proxy.conf +18 -0
  25. data/server/files/etc/apache2/sites-available/app.common +61 -0
  26. data/server/files/etc/apache2/sites-available/app.custom +0 -0
  27. data/server/files/etc/apache2/sites-available/default +14 -0
  28. data/server/files/etc/apache2/sites-available/default-ssl +19 -0
  29. data/server/files/etc/cron.d/ec2onrails +21 -0
  30. data/server/files/etc/cron.daily/app +24 -0
  31. data/server/files/etc/cron.daily/logrotate_post +19 -0
  32. data/server/files/etc/cron.hourly/app +24 -0
  33. data/server/files/etc/cron.monthly/app +24 -0
  34. data/server/files/etc/cron.weekly/app +24 -0
  35. data/server/files/etc/denyhosts.conf +628 -0
  36. data/server/files/etc/dpkg/dpkg.cfg +13 -0
  37. data/server/files/etc/ec2onrails/README +32 -0
  38. data/server/files/etc/ec2onrails/balancer_members +6 -0
  39. data/server/files/etc/ec2onrails/roles.yml +5 -0
  40. data/server/files/etc/environment +2 -0
  41. data/server/files/etc/god/app.god +40 -0
  42. data/server/files/etc/god/db.god +17 -0
  43. data/server/files/etc/god/dkim_filter.god +20 -0
  44. data/server/files/etc/god/examples/have_god_daemonize.god +18 -0
  45. data/server/files/etc/god/master.conf +35 -0
  46. data/server/files/etc/god/memcache.god +15 -0
  47. data/server/files/etc/god/notifications.god +14 -0
  48. data/server/files/etc/god/system.god +34 -0
  49. data/server/files/etc/god/web.god +38 -0
  50. data/server/files/etc/init.d/ec2-every-startup +29 -0
  51. data/server/files/etc/init.d/ec2-first-startup +36 -0
  52. data/server/files/etc/init.d/god +42 -0
  53. data/server/files/etc/init.d/nginx +78 -0
  54. data/server/files/etc/init.d/set_roles +3 -0
  55. data/server/files/etc/logrotate.d/apache2 +16 -0
  56. data/server/files/etc/logrotate.d/mongrel +11 -0
  57. data/server/files/etc/logrotate.d/nginx +11 -0
  58. data/server/files/etc/memcached.conf +47 -0
  59. data/server/files/etc/mongrel_cluster/app.yml +9 -0
  60. data/server/files/etc/motd.tail +13 -0
  61. data/server/files/etc/mysql/my.cnf +152 -0
  62. data/server/files/etc/nginx/nginx.conf +305 -0
  63. data/server/files/etc/postfix/main.cf +4 -0
  64. data/server/files/etc/rcS.d/S91ec2-first-startup +1 -0
  65. data/server/files/etc/rcS.d/S92ec2-every-startup +1 -0
  66. data/server/files/etc/rcS.d/S99set_roles +1 -0
  67. data/server/files/etc/ssh/sshd_config +94 -0
  68. data/server/files/etc/sudoers +1 -0
  69. data/server/files/etc/sudoers.full_access +26 -0
  70. data/server/files/etc/sudoers.restricted_access +28 -0
  71. data/server/files/etc/syslog.conf +69 -0
  72. data/server/files/usr/bin/god +26 -0
  73. data/server/files/usr/local/ec2onrails/COPYING +339 -0
  74. data/server/files/usr/local/ec2onrails/bin/archive_file.rb +44 -0
  75. data/server/files/usr/local/ec2onrails/bin/backup_app_db.rb +159 -0
  76. data/server/files/usr/local/ec2onrails/bin/ec2_meta_data.rb +80 -0
  77. data/server/files/usr/local/ec2onrails/bin/exec_runner +76 -0
  78. data/server/files/usr/local/ec2onrails/bin/init_services.rb +71 -0
  79. data/server/files/usr/local/ec2onrails/bin/optimize_mysql.rb +348 -0
  80. data/server/files/usr/local/ec2onrails/bin/rails_env +34 -0
  81. data/server/files/usr/local/ec2onrails/bin/rebundle.sh +70 -0
  82. data/server/files/usr/local/ec2onrails/bin/restore_app_db.rb +58 -0
  83. data/server/files/usr/local/ec2onrails/bin/set_rails_env +40 -0
  84. data/server/files/usr/local/ec2onrails/bin/set_roles.rb +87 -0
  85. data/server/files/usr/local/ec2onrails/bin/setup_web_proxy.rb +113 -0
  86. data/server/files/usr/local/ec2onrails/bin/update_hostname +40 -0
  87. data/server/files/usr/local/ec2onrails/config +30 -0
  88. data/server/files/usr/local/ec2onrails/lib/aws_helper.rb +76 -0
  89. data/server/files/usr/local/ec2onrails/lib/god_helper.rb +129 -0
  90. data/server/files/usr/local/ec2onrails/lib/god_patch.rb +43 -0
  91. data/server/files/usr/local/ec2onrails/lib/mysql_helper.rb +101 -0
  92. data/server/files/usr/local/ec2onrails/lib/roles_helper.rb +151 -0
  93. data/server/files/usr/local/ec2onrails/lib/s3_helper.rb +99 -0
  94. data/server/files/usr/local/ec2onrails/lib/utils.rb +16 -0
  95. data/server/files/usr/local/ec2onrails/lib/vendor/ini.rb +268 -0
  96. data/server/files/usr/local/ec2onrails/startup-scripts/every-startup/get-hostname.sh +23 -0
  97. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/README +5 -0
  98. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/create-dirs.sh +39 -0
  99. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/generate-default-web-cert-and-key.sh +49 -0
  100. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/misc.sh +27 -0
  101. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/prepare-mysql-data-dir.sh +24 -0
  102. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/setup-credentials.sh +29 -0
  103. data/server/files/usr/local/ec2onrails/startup-scripts/first-startup/setup-file-permissions.sh +30 -0
  104. data/server/rakefile.rb +249 -0
  105. data/setup.rb +1585 -0
  106. data/test/autobench.conf +60 -0
  107. data/test/spec/lib/s3_helper_spec.rb +134 -0
  108. data/test/spec/lib/s3_old.yml +3 -0
  109. data/test/spec/test_files/test1 +0 -0
  110. data/test/spec/test_files/test2 +0 -0
  111. data/test/test_app/Capfile +3 -0
  112. data/test/test_app/README +182 -0
  113. data/test/test_app/Rakefile +10 -0
  114. data/test/test_app/app/controllers/application.rb +7 -0
  115. data/test/test_app/app/controllers/db_fast_controller.rb +6 -0
  116. data/test/test_app/app/controllers/fast_controller.rb +5 -0
  117. data/test/test_app/app/controllers/slow_controller.rb +6 -0
  118. data/test/test_app/app/controllers/very_slow_controller.rb +6 -0
  119. data/test/test_app/app/helpers/application_helper.rb +3 -0
  120. data/test/test_app/app/helpers/db_fast_helper.rb +2 -0
  121. data/test/test_app/app/helpers/fast_helper.rb +2 -0
  122. data/test/test_app/app/helpers/slow_helper.rb +2 -0
  123. data/test/test_app/app/helpers/very_slow_helper.rb +2 -0
  124. data/test/test_app/config/boot.rb +109 -0
  125. data/test/test_app/config/database.yml +19 -0
  126. data/test/test_app/config/deploy.rb +21 -0
  127. data/test/test_app/config/environment.rb +60 -0
  128. data/test/test_app/config/environments/development.rb +21 -0
  129. data/test/test_app/config/environments/production.rb +18 -0
  130. data/test/test_app/config/environments/test.rb +19 -0
  131. data/test/test_app/config/routes.rb +27 -0
  132. data/test/test_app/db/schema.rb +7 -0
  133. data/test/test_app/doc/README_FOR_APP +2 -0
  134. data/test/test_app/public/404.html +30 -0
  135. data/test/test_app/public/500.html +30 -0
  136. data/test/test_app/public/dispatch.cgi +10 -0
  137. data/test/test_app/public/dispatch.fcgi +24 -0
  138. data/test/test_app/public/dispatch.rb +10 -0
  139. data/test/test_app/public/favicon.ico +0 -0
  140. data/test/test_app/public/images/rails.png +0 -0
  141. data/test/test_app/public/javascripts/application.js +2 -0
  142. data/test/test_app/public/javascripts/controls.js +963 -0
  143. data/test/test_app/public/javascripts/dragdrop.js +972 -0
  144. data/test/test_app/public/javascripts/effects.js +1120 -0
  145. data/test/test_app/public/javascripts/prototype.js +4225 -0
  146. data/test/test_app/public/robots.txt +1 -0
  147. data/test/test_app/script/about +3 -0
  148. data/test/test_app/script/breakpointer +3 -0
  149. data/test/test_app/script/console +3 -0
  150. data/test/test_app/script/destroy +3 -0
  151. data/test/test_app/script/generate +3 -0
  152. data/test/test_app/script/performance/benchmarker +3 -0
  153. data/test/test_app/script/performance/profiler +3 -0
  154. data/test/test_app/script/performance/request +3 -0
  155. data/test/test_app/script/plugin +3 -0
  156. data/test/test_app/script/process/inspector +3 -0
  157. data/test/test_app/script/process/reaper +3 -0
  158. data/test/test_app/script/process/spawner +3 -0
  159. data/test/test_app/script/runner +3 -0
  160. data/test/test_app/script/server +3 -0
  161. data/test/test_app/test/functional/db_fast_controller_test.rb +18 -0
  162. data/test/test_app/test/functional/fast_controller_test.rb +18 -0
  163. data/test/test_app/test/functional/slow_controller_test.rb +18 -0
  164. data/test/test_app/test/functional/very_slow_controller_test.rb +18 -0
  165. data/test/test_app/test/test_helper.rb +28 -0
  166. data/test/test_ec2onrails.rb +11 -0
  167. data/test/test_helper.rb +2 -0
  168. metadata +277 -0
@@ -0,0 +1,107 @@
1
+ # This is a sample Capistrano config file for EC2 on Rails.
2
+ # It should be edited and customized.
3
+
4
+ set :application, "yourapp"
5
+
6
+ set :repository, "http://svn.foo.com/svn/#{application}/trunk"
7
+
8
+ # NOTE: for some reason Capistrano requires you to have both the public and
9
+ # the private key in the same folder, the public key should have the
10
+ # extension ".pub".
11
+ ssh_options[:keys] = ["#{ENV['HOME']}/.ssh/your-ec2-key"]
12
+
13
+ # Your EC2 instances. Use the ec2-xxx....amazonaws.com hostname, not
14
+ # any other name (in case you have your own DNS alias) or it won't
15
+ # be able to resolve to the internal IP address.
16
+ role :web, "ec2-12-xx-xx-xx.z-1.compute-1.amazonaws.com"
17
+ role :app, "ec2-34-xx-xx-xx.z-1.compute-1.amazonaws.com"
18
+ role :memcache, "ec2-12-xx-xx-xx.z-1.compute-1.amazonaws.com"
19
+ role :db, "ec2-56-xx-xx-xx.z-1.compute-1.amazonaws.com", :primary => true
20
+ # role :db, "ec2-56-xx-xx-xx.z-1.compute-1.amazonaws.com", :primary => true, :ebs_vol_id => 'vol-12345abc'
21
+ # optinally, you can specify Amazon's EBS volume ID if the database is persisted
22
+ # via Amazon's EBS. See the main README for more information.
23
+
24
+ # Whatever you set here will be taken set as the default RAILS_ENV value
25
+ # on the server. Your app and your hourly/daily/weekly/monthly scripts
26
+ # will run with RAILS_ENV set to this value.
27
+ set :rails_env, "production"
28
+
29
+ # EC2 on Rails config.
30
+ # NOTE: Some of these should be omitted if not needed.
31
+ set :ec2onrails_config, {
32
+ # S3 bucket and "subdir" used by the ec2onrails:db:restore task
33
+ # NOTE: this only applies if you are not using EBS
34
+ :restore_from_bucket => "your-bucket",
35
+ :restore_from_bucket_subdir => "database",
36
+
37
+ # S3 bucket and "subdir" used by the ec2onrails:db:archive task
38
+ # This does not affect the automatic backup of your MySQL db to S3, it's
39
+ # just for manually archiving a db snapshot to a different bucket if
40
+ # desired.
41
+ # NOTE: this only applies if you are not using EBS
42
+ :archive_to_bucket => "your-other-bucket",
43
+ :archive_to_bucket_subdir => "db-archive/#{Time.new.strftime('%Y-%m-%d--%H-%M-%S')}",
44
+
45
+ # Set a root password for MySQL. Run "cap ec2onrails:db:set_root_password"
46
+ # to enable this. This is optional, and after doing this the
47
+ # ec2onrails:db:drop task won't work, but be aware that MySQL accepts
48
+ # connections on the public network interface (you should block the MySQL
49
+ # port with the firewall anyway).
50
+ # If you don't care about setting the mysql root password then remove this.
51
+ :mysql_root_password => "your-mysql-root-password",
52
+
53
+ # Any extra Ubuntu packages to install if desired
54
+ # If you don't want to install extra packages then remove this.
55
+ :packages => ["logwatch", "imagemagick"],
56
+
57
+ # Any extra RubyGems to install if desired: can be "gemname" or if a
58
+ # particular version is desired "gemname -v 1.0.1"
59
+ # If you don't want to install extra rubygems then remove this
60
+ # NOTE: if you are using rails 2.1, ec2onrails calls 'sudo rake gem:install',
61
+ # which will install gems defined in your rails configuration
62
+ :rubygems => ["rmagick", "rfacebook -v 0.9.7"],
63
+
64
+ # Defines the web proxy that will be used. Choices are :apache or :nginx
65
+ :web_proxy_server => :apache,
66
+
67
+ # extra security measures are taken if this is true, BUT it makes initial
68
+ # experimentation and setup a bit tricky. For example, if you do not
69
+ # have your ssh keys setup correctly, you will be locked out of your
70
+ # server after 3 attempts for upto 3 months.
71
+ :harden_server => false,
72
+
73
+ #if you want to harden the server, or setup email signing, you will need to set the domain
74
+ #if you use Capistrano's multistage extension (recommended!), you can add a line like this to your
75
+ #environment specific file:
76
+ # ec2onrails_config[:service_domain] = 'staging.mydomain.com'
77
+ :service_domain => nil,
78
+
79
+ # Set the server timezone. run "cap -e ec2onrails:server:set_timezone" for
80
+ # details
81
+ :timezone => "UTC",
82
+
83
+ # Files to deploy to the server (they'll be owned by root). It's intended
84
+ # mainly for customized config files for new packages installed via the
85
+ # ec2onrails:server:install_packages task. Subdirectories and files inside
86
+ # here will be placed in the same structure relative to the root of the
87
+ # server's filesystem.
88
+ # If you don't need to deploy customized config files to the server then
89
+ # remove this.
90
+ :server_config_files_root => "../server_configs",
91
+
92
+ # If config files are deployed, some services might need to be restarted.
93
+ # If you don't need to deploy customized config files to the server then
94
+ # remove this.
95
+ :services_to_restart => %w(postfix sysklogd),
96
+
97
+ # Set an email address to forward admin mail messages to. If you don't
98
+ # want to receive mail from the server (e.g. monit alert messages) then
99
+ # remove this.
100
+ :mail_forward_address => "you@yourdomain.com",
101
+
102
+ # Set this if you want SSL to be enabled on the web server. The SSL cert
103
+ # and key files need to exist on the server, The cert file should be in
104
+ # /etc/ssl/certs/default.pem and the key file should be in
105
+ # /etc/ssl/private/default.key (see :server_config_files_root).
106
+ :enable_ssl => true
107
+ }
data/examples/s3.yml ADDED
@@ -0,0 +1,9 @@
1
+ staging:
2
+ aws_access_key: ABC123
3
+ aws_secret_access_key: abc123abc123abc123abc123
4
+ bucket_base_name: yourbucket
5
+
6
+ production:
7
+ aws_access_key: DEF456
8
+ aws_secret_access_key: def456def456def456def456
9
+ bucket_base_name: yourbucket
@@ -0,0 +1,43 @@
1
+ module Ec2onrails
2
+ module CapistranoUtils
3
+ def run_local(command)
4
+ result = system command
5
+ raise("error: #{$?}") unless result
6
+ end
7
+
8
+ def run_init_script(script, arg)
9
+ # since init scripts might have the execute bit unset by the set_roles script we need to check
10
+ sudo "sh -c 'if [ -x /etc/init.d/#{script} ] ; then /etc/init.d/#{script} #{arg}; fi'"
11
+ end
12
+
13
+ # return hostnames for the role named role_sym that has the specified options
14
+ def hostnames_for_role(role_sym, options = {})
15
+ role = roles[role_sym]
16
+ unless role
17
+ return []
18
+ end
19
+ # make sure we match the server with all the passed in options, BUT the server can
20
+ # have additional options defined. e.g.: :primary => true and :ebs_vol_id => 'vol-1234abcd'
21
+ # but we want to select the server where :primary => true
22
+ role.select{|s|
23
+ match = true
24
+ options.each_pair{|k,v| match = false if s.options[k] != v}
25
+ }.collect{|s| s.host}
26
+ end
27
+
28
+ # Like the capture method, but does not print out error stream and swallows
29
+ # an exception if the process's exit code != 0
30
+ def quiet_capture(command, options={})
31
+ output = ""
32
+ invoke_command(command, options.merge(:once => true)) do |ch, stream, data|
33
+ case stream
34
+ when :out then output << data
35
+ # when :err then warn "[err :: #{ch[:server]}] #{data}"
36
+ end
37
+ end
38
+ ensure
39
+ return (output || '').strip
40
+ end
41
+
42
+ end
43
+ end
@@ -0,0 +1,359 @@
1
+ Capistrano::Configuration.instance.load do
2
+ cfg = ec2onrails_config
3
+
4
+ namespace :ec2onrails do
5
+ desc <<-DESC
6
+ Deploy and restore database from S3
7
+ DESC
8
+ task :restore_db_and_deploy do
9
+ db.recreate
10
+ deploy.update_code
11
+ deploy.symlink
12
+ db.restore
13
+ deploy.migrations
14
+ end
15
+
16
+ namespace :db do
17
+ desc <<-DESC
18
+ [internal] Load configuration info for the database from
19
+ config/database.yml, and start mysql (it must be running
20
+ in order to interact with it).
21
+ DESC
22
+ task :load_config do
23
+ unless hostnames_for_role(:db, :primary => true).empty?
24
+ db_config = YAML::load(ERB.new(File.read("config/database.yml")).result)[rails_env.to_s] || {}
25
+ cfg[:db_name] ||= db_config['database']
26
+ cfg[:db_user] ||= db_config['username'] || db_config['user']
27
+ cfg[:db_password] ||= db_config['password']
28
+ cfg[:db_host] ||= db_config['host']
29
+ cfg[:db_port] ||= db_config['port']
30
+ cfg[:db_socket] ||= db_config['socket']
31
+
32
+ if (cfg[:db_host].nil? || cfg[:db_host].empty?) && (cfg[:db_socket].nil? || cfg[:db_socket].empty?)
33
+ raise "ERROR: missing database config. Make sure database.yml contains a '#{rails_env}' section with either 'host: hostname' or 'socket: /var/run/mysqld/mysqld.sock'."
34
+ end
35
+
36
+ [cfg[:db_name], cfg[:db_user], cfg[:db_password]].each do |s|
37
+ if s.nil? || s.empty?
38
+ raise "ERROR: missing database config. Make sure database.yml contains a '#{rails_env}' section with a database name, user, and password."
39
+ elsif s.match(/['"]/)
40
+ raise "ERROR: database config string '#{s}' contains quotes."
41
+ end
42
+ end
43
+ end
44
+ end
45
+
46
+ desc <<-DESC
47
+ Create the MySQL database. Assumes there is no MySQL root \
48
+ password. To create a MySQL root password create a task that's run \
49
+ after this task using an after hook.
50
+ DESC
51
+ task :create, :roles => :db do
52
+ on_rollback { drop }
53
+ load_config
54
+ start
55
+ sleep(5) #make sure the db has some time to start up!
56
+
57
+
58
+ # remove the default test database, though sometimes it doesn't exist (perhaps it isn't there anymore?)
59
+ run %{mysql -u root -e "drop database if exists test; flush privileges;"}
60
+
61
+ # removing anonymous mysql accounts
62
+ run %{mysql -u root -D mysql -e "delete from db where User = ''; flush privileges;"}
63
+ run %{mysql -u root -D mysql -e "delete from user where User = ''; flush privileges;"}
64
+
65
+ # qoting of database names allows special characters eg (the-database-name)
66
+ # the quotes need to be double escaped. Once for capistrano and once for the host shell
67
+ run %{mysql -u root -e "create database if not exists \\`#{cfg[:db_name]}\\`;"}
68
+ run %{mysql -u root -e "grant all on \\`#{cfg[:db_name]}\\`.* to '#{cfg[:db_user]}'@'%' identified by '#{cfg[:db_password]}';"}
69
+ run %{mysql -u root -e "grant reload on *.* to '#{cfg[:db_user]}'@'%' identified by '#{cfg[:db_password]}';"}
70
+ run %{mysql -u root -e "grant super on *.* to '#{cfg[:db_user]}'@'%' identified by '#{cfg[:db_password]}';"}
71
+ end
72
+
73
+ desc <<-DESC
74
+ Move the MySQL database to Amazon's Elastic Block Store (EBS), \
75
+ which is a persistant data store for the cloud.
76
+ OPTIONAL PARAMETERS:
77
+ * SIZE: Pass in a number representing the GB's to hold, like 10. \
78
+ It will default to 10 gigs.
79
+ * VOLUME_ID: The volume_id to use for the mysql database
80
+ NOTE: keep track of the volume ID, as you'll want to keep this for your \
81
+ records and probably add it to the :db role in your deploy.rb file \
82
+ (see the ec2onrails sample deploy.rb file for additional information)
83
+ DESC
84
+ task :enable_ebs, :roles => :db, :only => { :primary => true } do
85
+ # based off of Eric's work:
86
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1663&categoryID=100
87
+ #
88
+ # EXPLAINATION:
89
+ # There is a lot going on here! At the end, the setup should be:
90
+ # * create EBS volume if run outside of the ec2onrails:setup and
91
+ # VOLUME_ID is not passed in when the cap task is called
92
+ # * EBS volume attached to /dev/sdh
93
+ # * format to xfs if new or do a xfs_check if previously existed
94
+ # * mounted on /var/local and update /etc/fstab
95
+ # * move /mnt/mysql_data -> /var/local/mysql_data
96
+ # * move /mnt/log/mysql -> /var/local/log/mysql
97
+ # * change mysql configs by writing /etc/mysql/conf.d/mysql-ec2-ebs.cnf
98
+ # * keep a copy of the mysql configs with the EBS volume, and if that volume is hooked into
99
+ # another instance, make sure the mysql configs that go with that volume are symlinked to /etc/mysql
100
+ # * update the file locations of the mysql binary logs in /mnt/log/mysql/mysql-bin.index
101
+ # * symlink the moved folders to their old position... makes the move to EBS transparent
102
+ # * Amazon doesn't contain EBS information in the meta-data API (yet). So write
103
+ # /etc/ec2onrails/ebs_info.yml
104
+ # to contain the meta-data information that we need
105
+ #
106
+ # DESIGN CONSIDERATIONS
107
+ # * only moving mysql data to EBS. seems the most obvious, and if we move over other components
108
+ # we will have to share that bandwidth (1 Gbps pipe to SAN). So limiting to what we really need
109
+ # * not moving all mysql logic over (tmp scratch space stays local). Again, this is to limit
110
+ # unnecessary bandwidth usage, PLUS, we are charged per million IO to EBS
111
+ #
112
+ # TODO:
113
+ # * make sure if we have a predefined ebs_vol_id, that we error out with a nice msg IF the zones do not match
114
+ # * can we move more of the mysql cache files back to the local disk and off of EBS, like the innodb table caches?
115
+ # * right now we force this task to only be run on one server; that works for db :primary => true
116
+ # But what is the best way to make this work if it needs to setup multiple servers (like db slaves)?
117
+ # I need to figure out how to do a direct mapping from a server definition to a ebs_vol_id
118
+ # * when we enable slaves and we setup ebs volumes on them, make it transparent to the user.
119
+ # have the slave create a snapshot of the db.master volume, and then use that to mount from
120
+ # * need to do a rollback that if the volume is created but something fails, lets uncreate it?
121
+ # carefull though! If it fails towards the end when information is copied over, it could cause information
122
+ # to be lost!
123
+ #
124
+
125
+ mysql_dir_root = '/var/local'
126
+ block_mnt = '/dev/sdh'
127
+ servers = find_servers_for_task(current_task)
128
+
129
+ if servers.empty?
130
+ raise Capistrano::NoMatchingServersError, "`#{task.fully_qualified_name}' is only run for servers matching #{task.options.inspect}, but no servers matched"
131
+ elsif servers.size > 1
132
+ raise Capistrano::Error, "`#{task.fully_qualified_name}' is can only be run on one server, not #{server.size}"
133
+ end
134
+
135
+ vol_id = ENV['VOLUME_ID'] || servers.first.options[:ebs_vol_id]
136
+
137
+ #HACK! capistrano doesn't allow arguments to be passed in if we call this task as a method, like 'db.enable_ebs'
138
+ # the places where we do call it like that, we don't want to force a move to ebs, so....
139
+ # if the call frame is > 1 (ie, another task called it), do NOT force the ebs move
140
+ no_force = task_call_frames.size > 1
141
+ prev_created = !(vol_id.nil? || vol_id.empty?)
142
+ #no vol_id was passed in, but perhaps it is already mounted...?
143
+ prev_created = true if !quiet_capture("mount | grep -inr '#{mysql_dir_root}' || echo ''").empty?
144
+
145
+ unless no_force && (vol_id.nil? || vol_id.empty?)
146
+ zone = quiet_capture("/usr/local/ec2onrails/bin/ec2_meta_data.rb -key 'placement/availability-zone'")
147
+ instance_id = quiet_capture("/usr/local/ec2onrails/bin/ec2_meta_data.rb -key 'instance-id'")
148
+
149
+ unless prev_created
150
+ puts "creating new ebs volume...."
151
+ size = ENV["SIZE"] || "10"
152
+ cmd = "/root/ec2-tools/bin/ec2-create-volume -s #{size} -z #{zone} 2>&1"
153
+ puts "running: #{cmd}"
154
+ output = `#{cmd}`
155
+ puts output
156
+ vol_id = (output =~ /^VOLUME\t(.+?)\t/ && $1)
157
+ puts "NOTE: remember that vol_id"
158
+ sleep(2)
159
+ end
160
+ vol_id.strip! if vol_id
161
+ if quiet_capture("mount | grep -inr '#{block_mnt}' || echo ''").empty?
162
+ cmd = "/root/ec2-tools/bin/ec2-attach-volume -d #{block_mnt} -i #{instance_id} #{vol_id} 2>&1"
163
+ puts "running: #{cmd}"
164
+ output = `#{cmd}`
165
+ puts output
166
+ if output =~ /Client.InvalidVolume.ZoneMismatch/i
167
+ raise Exception, "The volume you are trying to attach does not reside in the zone of your instance. Stopping!"
168
+ end
169
+ while !system( "/root/ec2-tools/bin/ec2-describe-volumes | grep #{vol_id} | grep attached" )
170
+ puts "Waiting for #{vol_id} to be attached..."
171
+ sleep 1
172
+ end
173
+ end
174
+
175
+ ec2onrails.server.allow_sudo do
176
+ # try to format the volume... if it is already formatted, lets run a check on
177
+ # it to make sure it is ok, and then continue on
178
+ # if errors, the device is busy...something else is going on here and it is already mounted... skip!
179
+ if prev_created
180
+ # Stop the db (mysql server) for cases where this is being run after the original run
181
+ # If EBS partiion is already mounted and being used by mysql, it will fail when umount is run
182
+ god_status = quiet_capture("sudo god status")
183
+ god_status = god_status.empty? ? {} : YAML::load(god_status)
184
+ start_stop_db = false
185
+ start_stop_db = god_status['db']['mysql'] == 'up'
186
+ if start_stop_db
187
+ stop
188
+ puts "Waiting for mysql to stop"
189
+ sleep(10)
190
+ end
191
+ quiet_capture("sudo umount #{mysql_dir_root}") #unmount if need to
192
+ puts "Checking if the filesystem needs to be created (if you created #{vol_id} yourself)"
193
+ existing = quiet_capture( "mkfs.xfs /dev/sdh", :via => 'sudo' ).match( /existing filesystem/ )
194
+ sudo "xfs_check #{block_mnt}"
195
+ # Restart the db if it
196
+ start if start_stop_db && existing
197
+ else
198
+ sudo "mkfs.xfs #{block_mnt}"
199
+ end
200
+
201
+ # if not added to /etc/fstab, lets do so
202
+ sudo "sh -c \"grep -iqn '#{mysql_dir_root}' /etc/fstab || echo '#{block_mnt} #{mysql_dir_root} xfs noatime 0 0' >> /etc/fstab\""
203
+ sudo "mkdir -p #{mysql_dir_root}"
204
+ #if not already mounted, lets mount it
205
+ sudo "sh -c \"mount | grep -iqn '#{mysql_dir_root}' || mount '#{mysql_dir_root}'\""
206
+
207
+ #ok, now lets move the mysql stuff off of /mnt -> mysql_dir_root
208
+ stop rescue nil #already stopped
209
+ sudo "mkdir -p #{mysql_dir_root}/log"
210
+ #move the data over, but keep a symlink to the new location for backwards compatibility
211
+ #and do not do it if /mnt/mysql_data has already been moved
212
+ quiet_capture("sudo sh -c 'test ! -d #{mysql_dir_root}/mysql_data && mv /mnt/mysql_data #{mysql_dir_root}/'")
213
+ sudo "mv /mnt/mysql_data /mnt/mysql_data_old 2>/dev/null || echo"
214
+ sudo "ln -fs #{mysql_dir_root}/mysql_data /mnt/mysql_data"
215
+
216
+ #but keep the tmpdir on mnt
217
+ sudo "sh -c 'mkdir -p /mnt/tmp/mysql && chown mysql:mysql /mnt/tmp/mysql'"
218
+ #move the logs over, but keep a symlink to the new location for backwards compatibility
219
+ #and do not do it if the logs have already been moved
220
+ quiet_capture("sudo sh -c 'test ! -d #{mysql_dir_root}/log/mysql_data && mv /mnt/log/mysql #{mysql_dir_root}/log/'")
221
+ sudo "ln -fs #{mysql_dir_root}/log/mysql /mnt/log/mysql"
222
+ quiet_capture("sudo sh -c \"test -f #{mysql_dir_root}/log/mysql/mysql-bin.index && \
223
+ perl -pi -e 's%/mnt/log/%#{mysql_dir_root}/log/%' #{mysql_dir_root}/log/mysql/mysql-bin.index\"") rescue false
224
+
225
+ if quiet_capture("test -d /var/local/etc/mysql && echo 'yes'").empty?
226
+ txt = <<-FILE
227
+ [mysqld]
228
+ datadir = #{mysql_dir_root}/mysql_data
229
+ tmpdir = /mnt/tmp/mysql
230
+ log_bin = #{mysql_dir_root}/log/mysql/mysql-bin.log
231
+ log_slow_queries = #{mysql_dir_root}/log/mysql/mysql-slow.log
232
+ FILE
233
+ put txt, '/tmp/mysql-ec2-ebs.cnf'
234
+ sudo 'mv /tmp/mysql-ec2-ebs.cnf /etc/mysql/conf.d/mysql-ec2-ebs.cnf'
235
+
236
+ #keep a copy
237
+ sudo "rsync -aR /etc/mysql #{mysql_dir_root}/"
238
+ end
239
+ # lets use the mysql configs on the EBS volume
240
+ sudo "mv /etc/mysql /etc/mysql.orig 2>/dev/null"
241
+ sudo "ln -sf #{mysql_dir_root}/etc/mysql /etc/mysql"
242
+
243
+ #just put a README on the drive so we know what this volume is for!
244
+ txt = <<-FILE
245
+ This volume is setup to be used by Ec2onRails in conjunction with Amazon's EBS, for primary MySql database persistence.
246
+ RAILS_ENV: #{fetch(:rails_env, 'undefined')}
247
+ DOMAIN: #{fetch(:domain, 'undefined')}
248
+
249
+ Modify this volume at your own risk
250
+ FILE
251
+
252
+ put txt, "/tmp/VOLUME-README"
253
+ sudo "mv /tmp/VOLUME-README #{mysql_dir_root}/VOLUME-README"
254
+ sudo "touch /etc/ec2onrails/ebs_info.yml"
255
+ ebs_info = quiet_capture("cat /etc/ec2onrails/ebs_info.yml")
256
+
257
+ ebs_info = ebs_info.empty? ? {} : YAML::load(ebs_info)
258
+ ebs_info[mysql_dir_root] = {'block_loc' => block_mnt, 'volume_id' => vol_id}
259
+ put(ebs_info.to_yaml, "/tmp/ebs_info.yml")
260
+ sudo "mv /tmp/ebs_info.yml /etc/ec2onrails/ebs_info.yml"
261
+ #lets start it back up
262
+ start
263
+ end #end of sudo
264
+ end
265
+ end
266
+
267
+
268
+ desc <<-DESC
269
+ [internal] Make sure the MySQL server has been started, just in case the db role
270
+ hasn't been set, e.g. when called from ec2onrails:setup.
271
+ (But don't enable monitoring on it.)
272
+ DESC
273
+ task :start, :roles => :db do
274
+ sudo "god start db"
275
+ end
276
+
277
+ task :stop, :roles => :db do
278
+ sudo "god stop db"
279
+ end
280
+
281
+
282
+ desc <<-DESC
283
+ Drop the MySQL database. Assumes there is no MySQL root \
284
+ password. If there is a MySQL root password, create a task that removes \
285
+ it and run that task before this one using a before hook.
286
+ DESC
287
+ task :drop, :roles => :db do
288
+ load_config
289
+ run %{mysql -u root -e "drop database if exists \\`#{cfg[:db_name]}\\`;"}
290
+ end
291
+
292
+ desc <<-DESC
293
+ db:drop and db:create.
294
+ DESC
295
+ task :recreate, :roles => :db do
296
+ drop
297
+ create
298
+ end
299
+
300
+ desc <<-DESC
301
+ Set a root password for MySQL, using the variable mysql_root_password \
302
+ if it is set. If this is done db:drop won't work.
303
+ DESC
304
+ task :set_root_password, :roles => :db do
305
+ if cfg[:mysql_root_password]
306
+ run %{mysql -u root -e "UPDATE mysql.user SET Password=PASSWORD('#{cfg[:mysql_root_password]}') WHERE User='root'; FLUSH PRIVILEGES;"}
307
+ end
308
+ end
309
+
310
+ desc <<-DESC
311
+ Dump the MySQL database to ebs (if enabled) or the S3 bucket specified by \
312
+ ec2onrails_config[:archive_to_bucket]. The filename will be \
313
+ "database-archive/<timestamp>/dump.sql.gz".
314
+ DESC
315
+ task :archive, :roles => :db do
316
+ run "/usr/local/ec2onrails/bin/backup_app_db.rb --bucket #{cfg[:archive_to_bucket]} --dir #{cfg[:archive_to_bucket_subdir]}"
317
+ end
318
+
319
+ desc <<-DESC
320
+ Restore the MySQL database from the S3 bucket specified by \
321
+ ec2onrails_config[:restore_from_bucket]. The archive filename is \
322
+ expected to be the default, "mysqldump.sql.gz".
323
+ DESC
324
+ task :restore, :roles => :db do
325
+ run "/usr/local/ec2onrails/bin/restore_app_db.rb --bucket #{cfg[:restore_from_bucket]} --dir #{cfg[:restore_from_bucket_subdir]}"
326
+ end
327
+
328
+ desc <<-DESC
329
+ [internal] Initialize the default backup folder on S3 (i.e. do a full
330
+ backup of the newly-created db so the automatic incremental backups
331
+ make sense). NOTE: Only of use if you do not have ebs enabled
332
+ DESC
333
+ task :init_backup, :roles => :db do
334
+ server.allow_sudo do
335
+ sudo "/usr/local/ec2onrails/bin/backup_app_db.rb --reset"
336
+ end
337
+ end
338
+
339
+ # do NOT run if the flag does not exist. This is placed by a startup script
340
+ # and it is only run on the first-startup. This means after the db has been
341
+ # optimized, this task will not work again.
342
+ #
343
+ # Of course you can overload it or call the file directly
344
+ task :optimize, :roles => :db do
345
+ if !quiet_capture("test -e /tmp/optimize_db_flag && echo 'file exists'").empty?
346
+ begin
347
+ sudo "/usr/local/ec2onrails/bin/optimize_mysql.rb"
348
+ ensure
349
+ sudo "rm -rf /tmp/optimize_db_flag" #remove so we cannot run again
350
+ end
351
+ else
352
+ puts "skipping as it looks like this task has already been run"
353
+ end
354
+ end
355
+
356
+ end
357
+
358
+ end
359
+ end
@@ -0,0 +1,30 @@
1
+ Capistrano::Configuration.instance(:must_exist).load do
2
+ cfg = ec2onrails_config
3
+
4
+ # override default start/stop/restart tasks to use god
5
+ namespace :deploy do
6
+ desc <<-DESC
7
+ Overrides the default Capistrano deploy:start, uses \
8
+ 'god start app'
9
+ DESC
10
+ task :start, :roles => :app do
11
+ sudo "god start app"
12
+ end
13
+
14
+ desc <<-DESC
15
+ Overrides the default Capistrano deploy:stop, uses \
16
+ 'god stop app'
17
+ DESC
18
+ task :stop, :roles => :app do
19
+ sudo "god stop app"
20
+ end
21
+
22
+ desc <<-DESC
23
+ Overrides the default Capistrano deploy:restart, uses \
24
+ 'god restart app'
25
+ DESC
26
+ task :restart, :roles => :app do
27
+ sudo "god restart app"
28
+ end
29
+ end
30
+ end