matross 0.3.0 → 0.4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YWRiYWQ4NDg4NGNlNDc0OWY5Y2M5NGU1NGViNTY3NWMxN2JmYzU1NQ==
+    ZDdiOTQ1YzRjMmRlOTRhZDk3YTdlNzc1NDhkMjhjMTJjNjY4YjYyOA==
   data.tar.gz: !binary |-
-    NDlkYmU4ZjNhNTFmNGNlMWJmMzMyMDllNTcyODIyNGNkODI3ZjVkOQ==
+    ZDhlYmYxZjhhODE2MTM5ODkxODA0NmFlNzkwZTRhOWIxMGQyODNmZA==
 SHA512:
   metadata.gz: !binary |-
-    YzJlMjY4MmE5ODkxYWI2YjE4N2I3MDU2NjA4NWY3YjliMzBmN2U5OTg0NjQx
-    ZjM0ODEwYmFkZDkwNDM3NWU4MTQwMTU2MzNkNWVjMDllMjBlYWNhMGY0OGYx
-    YTMxOTk2ZGU1ZjkxNzUzZjI4YTc3Zjg0YTE0OGJlNjZlNDQyOTE=
+    NTYzZTk0ZDAzZmU1ODhiYWJiYWNlOWZjMDBjN2FhYTk0ZmVmMzg0YTIwMmVh
+    ZDA2NTRiYzE0M2ZiOTExODNjNmNlM2NjNzFiN2RjZDUyMDBhYTIzNGEwNTVl
+    MjNjOTZkNzQzYzNjZjVhYjk5YzEyMTQ2ZjI2OTAzOWVmMTZkZjU=
   data.tar.gz: !binary |-
-    NjlhNzdhY2M0NDRlMzQwMjJkNTFiMWU0NTJjM2NjMzRiNzUxMzRhYmU4OWM2
-    M2I5ZTliOWFkNDJhYzE0N2Q1ZjZlODZmZmVkMmVhNDgzMmYxYjk3MzI1M2Zj
-    MGZlMDhkZGFiNTYxODIzZWU4NmFmZWM0ZTUwMzMyOWYwYTIxZjE=
+    YTY5NmNhN2M0NTI2NTk4NzVhNTY1OTM5YjA2OWFhZGQzN2Q4YjY2YzMwNGEw
+    YTRkOTZlZWU5MWM0NTE3Y2FiYjBmN2ViYWZmYmU2NTAyODdjMzkzZjAwMzcy
+    OTZlZWUyYTJjZmRlMzhmYTMxYWQ1NjQ0M2VhMmVmZDk1OWQ2MTk=

data/README.md

@@ -130,31 +130,41 @@ This recipes creates and configures the virtual_host for the application. [This
 | `nginx:lock`   | Sets up the a basic http auth on the virtual host |
 | `nginx:unlock` | Removes the basic http auth                       |
 
-
 ### MySQL
 
 Requires having [`mysql2`](http://rubygems.org/gems/mysql2) available in the application. In our MySQL recipe we dynamically generate a `database.yml` based on the variables that should be set globally or per-stage.
 
+The backup routine requires [`s3cmd`](http://s3tools.org/s3cmd) installed and properly configured. The key pair must have write access to the backup bucket on S3.
+
 Overwritable template: [`database.yml.erb`](lib/matross/templates/mysql/database.yml.erb)
 
 > Variables
 
-| Variable           | Default value                          | Description                                                                     |
-| ---                | ---                                    | ---                                                                             |
-| `:database_config` | `"#{shared_path}/config/database.yml"` | Location of the configuration file                                              |
-| `:mysql_host`      | None                                   | MySQL host address                                                              |
-| `:mysql_database`  | None                                   | MySQL database name. We automatically substitute dashes `-` for underscores `_` |
-| `:mysql_user`      | None                                   | MySQL user                                                                      |
-| `:mysql_passwd`    | None                                   | MySQL password                                                                  |
+| Variable                      | Default value                              | Description                                                      |
+| ---                           | ---                                        | ---                                                              |
+| `:database_config`            | `"#{shared_path}/config/database.yml"`     | Location of the configuration file                               |
+| `:mysql_host`                 | None                                       | MySQL host address                                               |
+| `:mysql_database`             | None                                       | MySQL database name. Dashes `-` are *gsubed* for underscores `_` |
+| `:mysql_user`                 | None                                       | MySQL user                                                       |
+| `:mysql_passwd`               | None                                       | MySQL password                                                   |
+| `:mysql_backup_script`        | `"#{shared_path}/matross/mysql_backup.sh"` | MySQL backup script location                                     |
+| `:mysql_backup_cron_schedule` | `'30 3 * * *'`                             | Cron schedule for the backup script                              |
+| `:mysql_backup_bucket`        | None                                       | Bucket used to store the dumps                                   |
+| `:mysql_backup_prefix`        | None                                       | Flat file prefix for the backup files                            |
 
 > Tasks
 
-| Task                | Description                                                         |
-| ---                 | ---                                                                 |
-| `mysql:setup`       | Creates the `database.yml` in the `shared_path`                     |
-| `mysql:symlink`     | Creates a symlink for the `database.yml` file in the `current_path` |
-| `mysql:create`      | Creates the database if it hasn't been created                      |
-| `mysql:schema_load` | Loads the schema if there are no tables in the DB                   |
+| Task                 | Description                                                                               |
+| ---                  | ---                                                                                       |
+| `mysql:setup`        | Creates the `database.yml` in the `shared_path`                                           |
+| `mysql:symlink`      | Creates a symlink for the `database.yml` file in the `current_path`                       |
+| `mysql:create`       | Creates the database if it hasn't been created                                            |
+| `mysql:schema_load`  | Loads the schema if there are no tables in the DB                                         |
+| `mysql:backup:setup` | Creates the backup script and configures the user's cron - *Not hooked to any other task* |
+| `mysql:dump:do`      | Dumps the application database                                                            |
+| `mysql:dump:get`     | Downloads a copy of the last generated database dump                                      |
+| `mysql:dump:apply`   | Apply the latest dump generated stored in 'dumps' locally                                 |
+
 
 ## Mongoid
 
@@ -198,7 +208,6 @@ Procfile task: `dj: bundle exec rake jobs:work` or `dj_<%= queue_name %>: bundle
 | ---                    | ---                                                               |
 | `delayed_job:procfile` | Defines how `delayed_job` should be run in a temporary `Procfile` |
 
-
 ### Fog (AWS)
 
 Requires having [`fog`](http://rubygems.org/gems/fog) available in the application. When we use `fog`, it is for interacting with Amazon services, once again very opinionated.

data/lib/matross/mysql.rb

@@ -1,6 +1,8 @@
 dep_included? 'mysql2'
 
-_cset(:database_config) { "#{shared_path}/config/database.yml" }
+_cset(:database_config)             { "#{shared_path}/config/database.yml" }
+_cset(:mysql_backup_script)         { "#{shared_path}/matross/mysql_backup.sh" }
+_cset :mysql_backup_cron_schedule,  '30 3 * * *'
 
 namespace :mysql do
 
@@ -18,26 +20,112 @@ namespace :mysql do
   after "bundle:install", "mysql:symlink"
 
   desc "Creates the application database"
-  task :create, :roles  => [:db] do
+  task :create, :roles  => :db do
     sql = <<-EOF.gsub(/^\s+/, '')
       CREATE DATABASE IF NOT EXISTS #{mysql_database.gsub("-", "_")};
     EOF
-    run "mysql --user=#{mysql_user} --password=#{mysql_passwd} --host=#{mysql_host} --execute=\"#{sql}\""
+    run %W{mysql --user=#{mysql_user}
+             #{'--password=' + mysql_passwd unless mysql_passwd.empty?}
+             --host=#{mysql_host}
+             --execute="#{sql}"} * ' '
   end
   after "mysql:setup", "mysql:create"
 
   desc "Loads the application schema into the database"
-  task :schema_load, :roles => [:db] do
+  task :schema_load, :roles => :db do
     sql = <<-EOF.gsub(/^\s+/, '')
       SELECT count(*) FROM information_schema.TABLES WHERE (TABLE_SCHEMA = '#{mysql_database.gsub("-", "_")}');
     EOF
-    table_count = capture("mysql --batch --skip-column-names "\
-                          "--user=#{mysql_user} "\
-                          "--password=#{mysql_passwd} "\
-                          "--host=#{mysql_host} "\
-                          "--execute=\"#{sql}\"").to_i
-    run "cd #{release_path} &&"\
-      "RAILS_ENV=#{rails_env.to_s.shellescape} bundle exec rake db:schema:load" if table_count == 0
+    table_count = capture(%W{mysql --batch --skip-column-names
+                              --user=#{mysql_user}
+                              #{'--password=' + mysql_passwd unless mysql_passwd.empty?}
+                              --host=#{mysql_host}
+                              --execute="#{sql}"} * ' ').to_i
+    run %W{cd #{release_path} &&
+           RAILS_ENV=#{rails_env.to_s.shellescape} bundle exec rake db:schema:load
+        } * ' ' if table_count == 0
   end
   after "mysql:symlink", "mysql:schema_load"
+
+  namespace :backup do
+
+    # This routine is heavily inspired by whenever's approach
+    # https://github.com/javan/whenever
+    desc "Updates the crontab"
+    task :setup, :roles => :db do
+      template "mysql/backup.sh.erb", mysql_backup_script
+      run "chmod +x #{mysql_backup_script}"
+
+      comment_open  = '# Begin Matross generated task for MySQL Backup'
+      comment_close = '# End Matross generated task for MySQL Backup'
+
+      cron_command = "#{mysql_backup_script} 2>&1 >> #{shared_path}/log/mysql_backup.log"
+      cron_entry   = "#{mysql_backup_cron_schedule} #{cron_command}"
+      cron         = [comment_open, cron_entry, comment_close].compact.join("\n")
+
+      current_crontab = ''
+      begin
+        # Some cron implementations require all non-comment lines to be
+        # newline-terminated. Strip all newlines and replace with the default
+        # platform record seperator ($/)
+        current_crontab = capture("crontab -l -u #{user} 2> /dev/null").gsub!(/\s+$/, $/)
+      rescue Capistrano::CommandError
+        logger.debug 'The user has no crontab'
+      end
+      contains_open_comment  = current_crontab =~ /^#{comment_open}\s*$/
+      contains_close_comment = current_crontab =~ /^#{comment_close}\s*$/
+
+      # If an existing identier block is found, replace it with the new cron entries
+      if contains_open_comment && contains_close_comment
+        updated_crontab = current_crontab.gsub(/^#{comment_open}\s*$.+^#{comment_close}\s*$/m, cron.chomp)
+      else  # Otherwise, append the new cron entries after any existing ones
+        updated_crontab = current_crontab.empty? ? cron : [current_crontab, cron].join("\n")
+      end.gsub(/\n{2,}/, "\n")  # More than one newline becomes just one.
+
+      temp_crontab_file = "/tmp/matross_#{user}_crontab"
+      put updated_crontab, temp_crontab_file
+      run "crontab -u #{user} #{temp_crontab_file}"
+      run "rm #{temp_crontab_file}"
+    end
+  end
+
+  namespace :dump do
+
+    desc "Dumps the application database"
+    task :do, :roles => :db, :except => { :no_release => true } do
+      run "mkdir -p #{shared_path}/dumps"
+      run %W{cd #{shared_path}/dumps &&
+             mysqldump --quote-names --create-options
+              --user=#{mysql_user}
+              #{'--password=' + mysql_passwd unless mysql_passwd.empty?}
+              --host=#{mysql_host}
+              #{mysql_database} |
+             gzip > "$(date +'#{mysql_database}_\%Y\%m\%d\%H\%M.sql.gz')"} * ' '
+    end
+
+    desc "Downloads a copy of the last generated database dump"
+    task :get, :roles => :db, :except => { :no_release => true } do
+      run_locally "mkdir -p dumps"
+      most_recent_bkp = capture(%W{find #{shared_path} -type f -name
+                                    '#{mysql_database}_*.sql.gz'} * ' '
+                               ).split.sort.last
+      abort "No dump found. Run mysql:dump:do." if most_recent_bkp.nil?
+
+      download most_recent_bkp, "dumps", :via => :scp
+      run_locally "gzip -d dumps/#{File.basename(most_recent_bkp)}"
+    end
+
+    desc "Apply the latest dump generated stored in 'dumps' locally"
+    task :apply, :roles => :db, :except => { :no_release => true } do
+      most_recent_bkp = %x[find dumps -type f -name\
+                            '#{mysql_database}_*.sql'].split.sort.last
+      abort "No dump found. Run mysql:dump:get." if most_recent_bkp.nil?
+
+      db_config = YAML.load(File.read('config/database.yml'))['development']
+      run_locally %W{mysql --user=#{db_config['username']}
+                      --host=#{db_config['host']}
+                      #{'--password=' + db_config['password'] unless db_config['password'].nil?}
+                      #{db_config['database']} < #{most_recent_bkp}} * ' '
+    end
+  end
 end

data/lib/matross/templates/mysql/backup.sh.erb

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+SRCDIR='/tmp/<%= application %>-mysql-backup'
+BUCKET=<%= mysql_backup_bucket %>
+PREFIX=<%= File.join(mysql_backup_prefix, '') %>
+
+# database access details
+HOST=<%= mysql_host %>
+USER=<%= mysql_user%>
+PASS=<%= mysql_passwd%>
+DB=<%= mysql_database%>
+
+# backup name
+BACKUP_NAME="$(date +'bkp%Y%m%d%H%M')"
+
+# make the temp directory if it doesn't exist and cd into it
+mkdir -p ${SRCDIR}
+cd ${SRCDIR}
+
+# mysql dump
+mysqldump -h${HOST} -u${USER} --force ${DB} > ${BACKUP_NAME}.sql
+tar --create --gzip --absolute-names --file ${BACKUP_NAME}.tar.gz ${BACKUP_NAME}.sql
+
+# upload to s3
+s3cmd put ${SRCDIR}/${BACKUP_NAME}.tar.gz s3://${BUCKET}/${PREFIX}
+
+# clean old files
+find ${SRCDIR}/* -mtime +5 -exec rm {} \;

data/lib/matross/version.rb

@@ -1,3 +1,3 @@
 module Matross
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: matross
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Artur Rodrigues
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-18 00:00:00.000000000 Z
+date: 2013-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,6 +84,7 @@ files:
 - lib/matross/templates/fog/fog_config.yml.erb
 - lib/matross/templates/foreman/process.conf.erb
 - lib/matross/templates/mongoid/mongoid.yml.erb
+- lib/matross/templates/mysql/backup.sh.erb
 - lib/matross/templates/mysql/database.yml.erb
 - lib/matross/templates/nginx/nginx_virtual_host_conf.erb
 - lib/matross/templates/unicorn/unicorn.rb.erb