scalingo_databases_rake_tasks 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9375caa0de1e6e6f5f2a7c0c2ac4d0a5a09a6357
+  data.tar.gz: 6dc2b750f6ddb3c52d8af993eb71c3aa57cfc12d
+SHA512:
+  metadata.gz: 4eabd1086dffe6a0afe83076e204debf67af3d2ed3e410dbc6d790ffc41345ec6765f996660d18b89d3ac350f89036355476f149aa61c36b39b0890531bd6787
+  data.tar.gz: f6b6d9ac88615b1a2130b5221a7a15c21f0dcf5bd1289e6e908c1975fda66891f437a5160aea337bdc66d4873d46a82c5d664d9874df0440001dffbd85521910

data/.gitignore

@@ -0,0 +1,39 @@
+# Created by https://www.gitignore.io/api/ruby
+
+### Ruby ###
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in scalingo_databases_rake_tasks.gemspec
+gemspec

data/README.md

@@ -0,0 +1,65 @@
+# ScalingoDatabasesRakeTasks
+
+A gem providing rake tasks such as **backup** and **restore** for database manipulations on [Scalingo](https://scalingo.com/).
+
+Currently supported databases:
+- MongoDB
+- MySQL
+- PostgreSQL
+
+Available tasks for **each** database:
+- `backup_local`: make a backup of local database
+- `backup_remote`: make a backup of remote Scalingo database
+- `restore_local`: restore a local database with an archive
+- `restore_remote`: restore a remote Scalingo database with an archive
+
+## Usage
+
+For remote operations you will have to set your shell's environment variable `APP` as your app name on Scalingo. The variable `DB_ENV_NAME` is optional, by default it will be the one generated when you provisionned the database addon. For example, if your database is a MongoDB then the variable will be `SCALINGO_MONGO_URL`.
+
+### Commands
+
+To see the complete list of tasks: `rake -T scalingo`
+
+Example of commands for MongoDB:
+- `rake scalingo:mongodb:backup_local`
+- `rake scalingo:mongodb:backup_remote`
+- `rake scalingo:mongodb:restore_local`
+- `rake scalingo:mongodb:restore_remote`
+
+### Backup and Restore
+
+Backups are stored under the `tmp` folder of your project, the database type is part of the archive name (e.g. scalingo_mongodb_dump.tar.gz).
+
+To restore from a specific archive, you'll have to give it the default archive name and put it inside of `tmp` folder before running the rake command.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'scalingo_databases_rake_tasks'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scalingo_databases_rake_tasks
+
+For Rails apps nothing to do.
+
+For other apps, add `require "scalingo_databases_rake_tasks"` to your Rakefile.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Scalingo/scalingo_databases_rake_tasks.
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+require "scalingo_databases_rake_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "scalingo_databases_rake_tasks"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/scalingo_databases_rake_tasks.rb

@@ -0,0 +1,18 @@
+require "scalingo_databases_rake_tasks/version"
+require "rake"
+
+module ScalingoDbTasks
+  include Rake::DSL if defined? Rake::DSL
+  def self.install_tasks
+    tasks = ["common", "mongodb", "mysql", "postgresql"]
+    tasks.each { |type| load("scalingo_databases_rake_tasks/tasks/#{type}.rake") }
+  end
+end
+
+# If we are in a Rails app use Railtie to load tasks.
+# Otherwise the user has to add `require "scalingo_databases_rake_tasks"` in Rakefile
+if defined?(Rails)
+  require "scalingo_databases_rake_tasks/railtie"
+else
+  ScalingoDbTasks::install_tasks
+end

data/lib/scalingo_databases_rake_tasks/railtie.rb

@@ -0,0 +1,12 @@
+require "rails"
+require "scalingo_databases_rake_tasks"
+
+module Rails
+  module ScalingoDbTasks
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        ::ScalingoDbTasks::install_tasks
+      end
+    end
+  end
+end

data/lib/scalingo_databases_rake_tasks/tasks/common.rake

@@ -0,0 +1,93 @@
+require 'uri'
+require 'open3'
+
+namespace :scalingo do
+  private
+
+  def confirm_remote(database)
+    print("You're about to restore your remote database #{database}.\nData will be destroyed. (y/N) ")
+    input = STDIN.gets.strip.downcase
+    if input != 'y'
+      abort
+    end
+  end
+
+  def archive_name name = "dump"
+    "#{Rails.root}/tmp/#{name}.tar.gz"
+  end
+
+  def make_tmp_dir
+    FileUtils.mkdir_p 'tmp'
+  end
+
+  def remote_credentials app, variable
+    output = `scalingo -a #{app} env | grep "^#{variable}=" | cut -d '=' -f2 | tr -d '\n'`
+    uri = URI(output.strip)
+    if uri.to_s.blank?
+      raise VariableError
+    else
+      [uri.path[1..-1], uri.user, uri.password]
+    end
+  end
+
+  def start_scalingo_tunnel app, variable
+    cmd = "scalingo -a #{app} db-tunnel -p 27717 #{variable}"
+    puts "*** Executing #{cmd}"
+    i, o, thr = Open3::pipeline_rw cmd
+
+    while true
+      read_line(o) do |line|
+        # puts line # debug
+        if line.include?("Encrypted")
+          abort "*** Your SSH key is encrypted. This gem is only compatible with SSH agents or unencrypted keys."
+        end
+
+        if line.include?("address already in use")
+          abort "*** Address 127.0.0.1:27717 is already in use."
+        end
+
+        if line.include?("'127.0.0.1:27717'")
+          return thr[0].pid
+        elsif line.include?("'127.0.0.1:")
+          abort "*** Address 127.0.0.1:27717 is already in use."
+        end
+      end
+    end
+  end
+
+  def read_line out
+    line = ""
+    while line == ""
+      sleep 0.2
+      begin
+        line = out.read_nonblock(200)
+        yield(line)
+      rescue EOFError, IO::EAGAINWaitReadable ; end
+    end
+  end
+
+  def open_tunnel default_env_name
+    app = ENV["APP"]
+    variable = ENV["DB_ENV_NAME"] || default_env_name
+    unless app
+      abort 'ENV["APP"] missing.'
+    end
+    unless variable
+      abort 'ENV["DB_ENV_NAME"] missing.'
+    end
+
+    database, user, password = remote_credentials(app, variable)
+    pid = start_scalingo_tunnel(app, variable)
+    at_exit do
+      Process.kill("INT", pid) if pid != -1
+      puts '*** Tunnel closed'
+    end
+    puts '*** Tunnel opened'
+    yield(database, user, password)
+  end
+
+  class VariableError < StandardError
+
+  end
+
+end

data/lib/scalingo_databases_rake_tasks/tasks/mongodb.rake

@@ -0,0 +1,106 @@
+namespace :scalingo do
+  namespace :mongodb do
+    desc "Backup local MongoDB database"
+    task :backup_local => :environment do
+      database, user, password, host = ScalingoMongoDB.local_credentials ENV['FILE']
+      ScalingoMongoDB.backup(database, user, password, host)
+    end
+
+    desc "Backup remote Scalingo MongoDB database"
+    task :backup_remote => :environment do
+      open_tunnel("SCALINGO_MONGO_URL") do |database, user, password|
+        ScalingoMongoDB.backup(database, user, password, "127.0.0.1:27717")
+      end
+    end
+
+    desc "Restore local MongoDB database using a Scalingo backup"
+    task :restore_local => :environment do
+      database, user, password, host = ScalingoMongoDB.local_credentials ENV['FILE']
+      ScalingoMongoDB.restore(database, user, password, host)
+    end
+
+    desc "Restore remote Scalingo MongoDB database using local backup"
+    task :restore_remote => :environment do
+      open_tunnel("SCALINGO_MONGO_URL") do |database, user, password|
+        confirm_remote(database)
+        ScalingoMongoDB.restore(database, user, password, "127.0.0.1:27717")
+      end
+    end
+
+    private
+
+    module ScalingoMongoDB
+      DUMP_NAME = "scalingo_mongodb_dump"
+      DUMP_PATH = Dir.tmpdir() + "/#{DUMP_NAME}"
+
+      def self.local_credentials(filename)
+        filename ||= "mongoid"
+        result = File.read "#{Rails.root}/config/#{filename}.yml"
+        config_file = YAML::load(ERB.new(result).result)
+
+        if config_file[Rails.env]['sessions']['default']['uri']
+          require 'uri'
+          uri = URI.parse config_file[Rails.env]['sessions']['default']['uri']
+
+          return [
+            uri.path[1..-1],
+            uri.user,
+            uri.password,
+            (uri.host || "127.0.0.1")
+          ]
+        else
+          return [
+            config_file[Rails.env]['sessions']['default']['database'],
+            config_file[Rails.env]['sessions']['default']['username'],
+            config_file[Rails.env]['sessions']['default']['password'],
+            config_file[Rails.env]['sessions']['default']['hosts'].first || "127.0.0.1"
+          ]
+        end
+      end
+
+      def self.backup database, user, password, host
+        cmd = "rm -rf #{DUMP_PATH} 2>/dev/null && /usr/bin/env mongodump -h #{host} -d #{database} -o #{DUMP_PATH}"
+        if user.blank?
+          output = cmd
+        else
+          cmd << " -u #{user}"
+          if password.blank?
+            output = cmd
+          else
+            cmd << " --password"
+            output = "#{cmd} ... [password filtered]"
+            cmd << " #{password}"
+          end
+        end
+
+        [cmd, output].each do |command|
+          command << " && tar czfh #{archive_name DUMP_NAME} -C #{Dir.tmpdir()} #{DUMP_NAME}"
+        end
+
+        puts "*** Executing #{output}"
+        make_tmp_dir
+        system(cmd)
+      end
+
+      def self.restore database, user, password, host
+        cmd = "rm -rf #{DUMP_PATH}/ 2>/dev/null && tar xvzf #{archive_name DUMP_NAME} -C #{Dir.tmpdir()}"
+        cmd << " && /usr/bin/env mongorestore --drop -h #{host} -d #{database} --dir #{DUMP_PATH}/*"
+        if user.blank?
+          output = cmd
+        else
+          cmd << " -u #{user}"
+          if password.blank?
+            output = cmd
+          else
+            cmd << " --password"
+            output = "#{cmd} ... [password filtered]"
+            cmd << " #{password}"
+          end
+        end
+
+        puts "*** Executing #{output}"
+        system(cmd)
+      end
+    end
+  end
+end

data/lib/scalingo_databases_rake_tasks/tasks/mysql.rake

@@ -0,0 +1,87 @@
+namespace :scalingo do
+  namespace :mysql do
+    desc "Backup local MySQL database"
+    task :backup_local => :environment do
+      database, user, password, host, port = ScalingoMySQL.local_credentials
+      ScalingoMySQL.backup(database, user, password, host, port)
+    end
+
+    desc "Backup remote Scalingo MySQL database"
+    task :backup_remote => :environment do
+      open_tunnel("SCALINGO_MYSQL_URL") do |database, user, password|
+        ScalingoMySQL.backup(database, user, password, "127.0.0.1", 27717)
+      end
+    end
+
+    desc "Restore local MySQL database using a Scalingo backup"
+    task :restore_local => :environment do
+      database, user, password, host, port = ScalingoMySQL.local_credentials
+      ScalingoMySQL.restore(database, user, password, host, port)
+    end
+
+    desc "Restore remote Scalingo MySQL database using local backup"
+    task :restore_remote => :environment do
+      open_tunnel("SCALINGO_MYSQL_URL") do |database, user, password|
+        confirm_remote(database)
+        ScalingoMySQL.restore(database, user, password, "127.0.0.1", 27717)
+      end
+    end
+
+    private
+
+    module ScalingoMySQL
+      DUMP_NAME = "scalingo_mysql_dump.sql"
+      DUMP_PATH = Dir.tmpdir() + "/#{DUMP_NAME}"
+
+      def self.local_credentials
+        config = ActiveRecord::Base.configurations[Rails.env]
+        return [
+          config['database'],
+          config['username'],
+          config['password'],
+          config['host'] || "127.0.0.1",
+          config['port'] || "3306",
+        ]
+      end
+
+      def self.backup database, user, password, host, port
+        base_cmd = "/usr/bin/env mysqldump --add-drop-table --create-options --disable-keys --extended-insert --single-transaction --quick --set-charset -h #{host} -P #{port} -u #{user}"
+        cmd = ""
+        cmd << base_cmd
+        public_cmd = ""
+        public_cmd << base_cmd
+
+        unless password.nil?
+          cmd << " -p'#{password}'"
+          public_cmd << " -p [password filtered]"
+        end
+        tar_cmd = " #{database} > #{DUMP_PATH} && tar cvzf #{archive_name DUMP_NAME} -C #{Dir.tmpdir()} #{DUMP_NAME}"
+        cmd << tar_cmd
+        public_cmd << tar_cmd
+
+        puts "*** Executing #{public_cmd}"
+        make_tmp_dir
+        system(cmd)
+      end
+
+      def self.restore database, user, password, host, port
+        base_cmd = "/usr/bin/env tar xvzOf #{archive_name DUMP_NAME} | /usr/bin/env mysql -h #{host} -P #{port} -u #{user}"
+        cmd = ""
+        cmd << base_cmd
+        public_cmd = ""
+        public_cmd << base_cmd
+
+        unless password.nil?
+          cmd << " -p'#{password}'"
+          public_cmd << " -p [password filtered]"
+        end
+        cmd << " #{database}"
+        public_cmd << " #{database}"
+
+        puts "*** Executing #{public_cmd}"
+        system(cmd)
+      end
+    end
+
+  end
+end

data/lib/scalingo_databases_rake_tasks/tasks/postgresql.rake

@@ -0,0 +1,90 @@
+namespace :scalingo do
+  namespace :postgresql do
+    desc "Backup local PostgreSQL database"
+    task :backup_local => :environment do
+      database, user, password, host, port = ScalingoPostgreSQL.local_credentials
+      ScalingoPostgreSQL.backup(database, user, password, host, port)
+    end
+
+    desc "Backup remote Scalingo PostgreSQL database"
+    task :backup_remote => :environment do
+      open_tunnel("SCALINGO_POSTGRESQL_URL") do |database, user, password|
+        ScalingoPostgreSQL.backup(database, user, password, "127.0.0.1", "27717")
+      end
+    end
+
+    desc "Restore local PostgreSQL database using a Scalingo backup"
+    task :restore_local => :environment do
+      database, user, password, host, port = ScalingoPostgreSQL.local_credentials
+      ScalingoPostgreSQL.restore(database, user, password, host, port)
+    end
+
+    desc "Restore remote Scalingo PostgreSQL database using local backup"
+    task :restore_remote => :environment do
+      open_tunnel("SCALINGO_POSTGRESQL_URL") do |database, user, password|
+        confirm_remote(database)
+        ScalingoPostgreSQL.restore(database, user, password, "127.0.0.1", "27717")
+      end
+    end
+
+    private
+
+    module ScalingoPostgreSQL
+      DUMP_NAME = "scalingo_postgresql_dump"
+      DUMP_PATH = Dir.tmpdir() + "/#{DUMP_NAME}"
+
+      def self.local_credentials
+        config = ActiveRecord::Base.configurations[Rails.env]
+        return [
+          config['database'],
+          config['username'],
+          config['password'],
+          config['host'] || "127.0.0.1",
+          config['port'] || "5432",
+        ]
+      end
+
+      def self.backup database, user, password, host, port
+        user_cmd = ""
+        password_cmd = ""
+        if not user.blank?
+          user_cmd = " -U #{user}"
+          if not password.blank?
+            password_cmd = "PGPASSWORD=#{password}"
+          end
+        end
+        base_cmd = "pg_dump -O -n public --format=c #{user_cmd} -h #{host} -p #{port} -d #{database}"
+        output = "rm -rf #{DUMP_PATH} 2>/dev/null && /usr/bin/env PGPASSWORD=[FILTERED] #{base_cmd}"
+        cmd = "rm -rf #{DUMP_PATH} 2>/dev/null && /usr/bin/env #{password_cmd} #{base_cmd}"
+
+        [cmd, output].each do |command|
+          command << " > #{DUMP_PATH} && tar cvzf #{archive_name DUMP_NAME} -C #{Dir.tmpdir()} #{DUMP_NAME}"
+        end
+
+        puts "*** Executing #{output}"
+        make_tmp_dir
+        system(cmd)
+      end
+
+      def self.restore database, user, password, host, port
+        user_cmd = ""
+        password_cmd = ""
+        if not user.blank?
+          user_cmd = " -U #{user}"
+          if not password.blank?
+            password_cmd = "PGPASSWORD=#{password}"
+          end
+        end
+
+        base_cmd = "tar xvzOf #{archive_name DUMP_NAME} | "
+        pg_cmd = "pg_restore -O -n public --clean --if-exists #{user_cmd} -h #{host} -p #{port} -d #{database}"
+        output = "#{base_cmd} PGPASSWORD=[FILTERED] #{pg_cmd}"
+        cmd = "#{base_cmd} #{password_cmd} #{pg_cmd}"
+
+        puts "*** Executing #{output}"
+        system(cmd)
+      end
+    end
+
+  end
+end

data/lib/scalingo_databases_rake_tasks/version.rb

@@ -0,0 +1,3 @@
+module ScalingoDbTasks
+  VERSION = "0.1.1"
+end

data/scalingo_databases_rake_tasks.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scalingo_databases_rake_tasks/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "scalingo_databases_rake_tasks"
+  spec.version       = ScalingoDbTasks::VERSION
+  spec.authors       = ["Scalingo"]
+  spec.email         = ["hello@scalingo.com"]
+
+  spec.summary       = %q{Perform database related tasks on Scalingo.}
+  spec.description   = %q{Perform database related tasks on Scalingo.}
+  spec.homepage      = "https://github.com/Scalingo/scalingo_databases_rake_tasks-gem"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: scalingo_databases_rake_tasks
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Scalingo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Perform database related tasks on Scalingo.
+email:
+- hello@scalingo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/scalingo_databases_rake_tasks.rb
+- lib/scalingo_databases_rake_tasks/railtie.rb
+- lib/scalingo_databases_rake_tasks/tasks/common.rake
+- lib/scalingo_databases_rake_tasks/tasks/mongodb.rake
+- lib/scalingo_databases_rake_tasks/tasks/mysql.rake
+- lib/scalingo_databases_rake_tasks/tasks/postgresql.rake
+- lib/scalingo_databases_rake_tasks/version.rb
+- scalingo_databases_rake_tasks.gemspec
+homepage: https://github.com/Scalingo/scalingo_databases_rake_tasks-gem
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Perform database related tasks on Scalingo.
+test_files: []