dump 1.0.0

data/lib/generators/assets_config/templates/assets

@@ -0,0 +1,8 @@
+# Lines starting with # are comments
+# Put paths of dirs you want to dump here
+# Example
+#   public/audios
+#   public/flash
+#   public/images/upload
+#   public/videos
+public/system

data/lib/tasks/assets.rake

@@ -0,0 +1,17 @@
+$: << File.join(File.dirname(__FILE__), '../../lib')
+require 'dump_rake'
+
+task :assets do
+  ENV['ASSETS'] ||= DumpRake::Assets.assets
+end
+
+namespace :assets do
+  desc "Delete assets" << DumpRake::Env.explain_variables_for_command(:assets)
+  task :delete => :assets do
+    ENV['ASSETS'].split(':').each do |asset|
+      DumpRake::Assets.glob_asset_children(asset, '*').each do |child|
+        FileUtils.remove_entry(child)
+      end
+    end
+  end
+end

data/lib/tasks/dump.rake

@@ -0,0 +1,27 @@
+$: << File.join(File.dirname(__FILE__), '../../lib')
+require 'dump_rake'
+
+desc "Short for dump:create" << DumpRake::Env.explain_variables_for_command(:create)
+task :dump => 'dump:create'
+
+namespace :dump do
+  desc "Show avaliable versions" << DumpRake::Env.explain_variables_for_command(:versions)
+  task :versions => :environment do
+    DumpRake.versions(DumpRake::Env.for_command(:versions))
+  end
+
+  desc "Create dump" << DumpRake::Env.explain_variables_for_command(:create)
+  task :create => :environment do
+    DumpRake.create(DumpRake::Env.for_command(:create))
+  end
+
+  desc "Restore dump" << DumpRake::Env.explain_variables_for_command(:restore)
+  task :restore => :environment do
+    DumpRake.restore(DumpRake::Env.for_command(:restore))
+  end
+
+  desc "Cleanup dumps" << DumpRake::Env.explain_variables_for_command(:cleanup)
+  task :cleanup => :environment do
+    DumpRake.cleanup(DumpRake::Env.for_command(:cleanup))
+  end
+end

data/recipes/dump.rb

@@ -0,0 +1,343 @@
+# encoding: UTF-8
+
+Capistrano::Configuration.instance(:i_need_this!).load do
+  $: << File.join(File.dirname(__FILE__), '../lib')
+  require 'fileutils'
+  require 'shellwords'
+
+  require 'dump_rake/continious_timeout'
+  require 'dump_rake/env'
+
+  begin
+    nil.blank?
+  rescue
+    Object.class_eval do
+      def blank?
+        respond_to?(:empty?) ? empty? : !self
+      end
+      def present?
+        !blank?
+      end
+    end
+  end
+
+  namespace :dump do
+    def dump_command(command, env = {})
+      rake = env.delete(:rake) || 'rake'
+
+      # stringify_keys! from activesupport
+      DumpRake::Env.stringify!(env)
+
+      env.update(DumpRake::Env.for_command(command, true))
+
+      cmd = %W[#{rake} -s dump:#{command}]
+      cmd += env.sort.map{ |key, value| "#{key}=#{value}" }
+      cmd.shelljoin
+    end
+
+    def fetch_rails_env
+      fetch(:rails_env, "production")
+    end
+
+    def got_rsync?
+      `which rsync`
+      $?.success?
+    end
+
+    def do_transfer_via(via, direction, from, to)
+      case via
+      when :rsync
+        if run_local('which rsync').present? && $?.success?
+          execute_on_servers do |servers|
+            commands = servers.map do |server|
+              target = sessions[server]
+              user = target.options[:user] || fetch(:user, nil)
+              host = target.host
+              port = target.options[:port]
+              full_host = "#{"#{user}@" if user.present?}#{host}"
+
+              ssh = port.present? ? "ssh -p #{port}" : 'ssh'
+              cmd = %W[rsync -P -e #{ssh} --timeout=15]
+              case direction
+              when :up
+                cmd << from << "#{full_host}:#{to}"
+              when :down
+                cmd << "#{full_host}:#{from}" << to
+              else
+                raise "Don't know how to transfer in direction #{direction}"
+              end
+              cmd.shelljoin
+            end
+            commands.each do |cmd|
+              logger.info cmd if logger
+
+              3.times do
+                break if system(cmd)
+                break unless [10, 11, 12, 23, 30, 35].include?($?.exitstatus)
+              end
+              raise "rsync returned #{$?.exitstatus}" unless $?.success?
+            end
+          end
+        end
+      when :sftp, :scp
+        ContiniousTimeout.timeout 15 do |thread|
+          transfer(direction, from, to, :via => via) do |channel, path, transfered, total|
+            thread.defer
+            progress = if transfered < total
+              "\e[1m%5.1f%%\e[0m" % (transfered * 100.0 / total)
+            else
+              "100%"
+            end
+            $stderr << "\rTransfering: #{progress}"
+          end
+        end
+      else
+        raise "Unknown transfer method #{via}"
+      end
+    end
+
+    def do_transfer(direction, from, to)
+      if via = DumpRake::Env[:transfer_via]
+        case via.downcase
+        when 'rsync'
+          do_transfer_via(:rsync, direction, from, to)
+        when 'sftp'
+          do_transfer_via(:sftp, direction, from, to)
+        when 'scp'
+          do_transfer_via(:scp, direction, from, to)
+        else
+          raise "Unknown transfer method #{via}"
+        end
+      else
+        if got_rsync?
+          do_transfer_via(:rsync, direction, from, to)
+        else
+          $stderr.puts "To transfer using rsync — make rsync binary accessible and verify that remote host can work with rsync through ssh"
+          begin
+            do_transfer_via(:sftp, direction, from, to)
+          rescue => e
+            $stderr.puts e
+            do_transfer_via(:scp, direction, from, to)
+          end
+        end
+      end
+    end
+
+    def with_additional_tags(*tags)
+      tags = [tags, DumpRake::Env[:tags]].flatten.select(&:present?).join(',')
+      DumpRake::Env.with_env(:tags => tags) do
+        yield
+      end
+    end
+
+    def print_and_return_or_fail
+      out = yield
+      raise 'Failed creating dump' if out.blank?
+      print out
+      out.strip
+    end
+
+    def run_local(cmd)
+      `#{cmd}`
+    end
+
+    def run_remote(cmd)
+      output = ''
+      run(cmd) do |channel, io, data|
+        case io
+        when :out
+          output << data
+        when :err
+          $stderr << data
+        end
+      end
+      output
+    end
+
+    def last_part_of_last_line(out)
+      if line = out.strip.split(/\s*[\n\r]\s*/).last
+        line.split("\t").last
+      end
+    end
+
+    def fetch_rake
+      fetch(:rake, nil)
+    end
+
+    def auto_backup?
+      !DumpRake::Env.no?(:backup)
+    end
+
+    namespace :local do
+      desc "Shorthand for dump:local:create" << DumpRake::Env.explain_variables_for_command(:create)
+      task :default, :roles => :db, :only => {:primary => true} do
+        create
+      end
+
+      desc "Create local dump" << DumpRake::Env.explain_variables_for_command(:create)
+      task :create, :roles => :db, :only => {:primary => true} do
+        print_and_return_or_fail do
+          with_additional_tags('local') do
+            run_local(dump_command(:create))
+          end
+        end
+      end
+
+      desc "Restore local dump" << DumpRake::Env.explain_variables_for_command(:restore)
+      task :restore, :roles => :db, :only => {:primary => true} do
+        run_local(dump_command(:restore))
+      end
+
+      desc "Versions of local dumps" << DumpRake::Env.explain_variables_for_command(:versions)
+      task :versions, :roles => :db, :only => {:primary => true} do
+        print run_local(dump_command(:versions, :show_size => true))
+      end
+
+      desc "Cleanup local dumps" << DumpRake::Env.explain_variables_for_command(:cleanup)
+      task :cleanup, :roles => :db, :only => {:primary => true} do
+        print run_local(dump_command(:cleanup))
+      end
+
+      desc "Upload dump" << DumpRake::Env.explain_variables_for_command(:transfer)
+      task :upload, :roles => :db, :only => {:primary => true} do
+        file = DumpRake::Env.with_env(:summary => nil) do
+          last_part_of_last_line(run_local(dump_command(:versions)))
+        end
+        if file
+          do_transfer :up, "dump/#{file}", "#{current_path}/dump/#{file}"
+        end
+      end
+    end
+
+    namespace :remote do
+      desc "Shorthand for dump:remote:create" << DumpRake::Env.explain_variables_for_command(:create)
+      task :default, :roles => :db, :only => {:primary => true} do
+        remote.create
+      end
+
+      desc "Create remote dump" << DumpRake::Env.explain_variables_for_command(:create)
+      task :create, :roles => :db, :only => {:primary => true} do
+        print_and_return_or_fail do
+          with_additional_tags('remote') do
+            run_remote("cd #{current_path}; #{dump_command(:create, :rake => fetch_rake, :RAILS_ENV => fetch_rails_env, :PROGRESS_TTY => '+')}")
+          end
+        end
+      end
+
+      desc "Restore remote dump" << DumpRake::Env.explain_variables_for_command(:restore)
+      task :restore, :roles => :db, :only => {:primary => true} do
+        run_remote("cd #{current_path}; #{dump_command(:restore, :rake => fetch_rake, :RAILS_ENV => fetch_rails_env, :PROGRESS_TTY => '+')}")
+      end
+
+      desc "Versions of remote dumps" << DumpRake::Env.explain_variables_for_command(:versions)
+      task :versions, :roles => :db, :only => {:primary => true} do
+        print run_remote("cd #{current_path}; #{dump_command(:versions, :rake => fetch_rake, :RAILS_ENV => fetch_rails_env, :PROGRESS_TTY => '+', :show_size => true)}")
+      end
+
+      desc "Cleanup of remote dumps" << DumpRake::Env.explain_variables_for_command(:cleanup)
+      task :cleanup, :roles => :db, :only => {:primary => true} do
+        print run_remote("cd #{current_path}; #{dump_command(:cleanup, :rake => fetch_rake, :RAILS_ENV => fetch_rails_env, :PROGRESS_TTY => '+')}")
+      end
+
+      desc "Download dump" << DumpRake::Env.explain_variables_for_command(:transfer)
+      task :download, :roles => :db, :only => {:primary => true} do
+        file = DumpRake::Env.with_env(:summary => nil) do
+          last_part_of_last_line(run_remote("cd #{current_path}; #{dump_command(:versions, :rake => fetch_rake, :RAILS_ENV => fetch_rails_env, :PROGRESS_TTY => '+')}"))
+        end
+        if file
+          FileUtils.mkpath('dump')
+          do_transfer :down, "#{current_path}/dump/#{file}", "dump/#{file}"
+        end
+      end
+    end
+
+    desc "Shorthand for dump:local:upload" << DumpRake::Env.explain_variables_for_command(:transfer)
+    task :upload, :roles => :db, :only => {:primary => true} do
+      local.upload
+    end
+
+    desc "Shorthand for dump:remote:download" << DumpRake::Env.explain_variables_for_command(:transfer)
+    task :download, :roles => :db, :only => {:primary => true} do
+      remote:download
+    end
+
+    namespace :mirror do
+      desc "Creates local dump, uploads and restores on remote" << DumpRake::Env.explain_variables_for_command(:mirror)
+      task :up, :roles => :db, :only => {:primary => true} do
+        auto_backup = if auto_backup?
+          with_additional_tags('auto-backup') do
+            remote.create
+          end
+        end
+        if !auto_backup? || auto_backup.present?
+          file = with_additional_tags('mirror') do
+            local.create
+          end
+          if file.present?
+            DumpRake::Env.with_clean_env(:like => file) do
+              local.upload
+              remote.restore
+            end
+          end
+        end
+      end
+
+      desc "Creates remote dump, downloads and restores on local" << DumpRake::Env.explain_variables_for_command(:mirror)
+      task :down, :roles => :db, :only => {:primary => true} do
+        auto_backup = if auto_backup?
+          with_additional_tags('auto-backup') do
+            local.create
+          end
+        end
+        if !auto_backup? || auto_backup.present?
+          file = with_additional_tags('mirror') do
+            remote.create
+          end
+          if file.present?
+            DumpRake::Env.with_clean_env(:like => file) do
+              remote.download
+              local.restore
+            end
+          end
+        end
+      end
+    end
+
+    namespace :backup do
+      desc "Shorthand for dump:backup:create" << DumpRake::Env.explain_variables_for_command(:backup)
+      task :default, :roles => :db, :only => {:primary => true} do
+        create
+      end
+
+      desc "Creates remote dump and downloads to local (desc defaults to 'backup')" << DumpRake::Env.explain_variables_for_command(:backup)
+      task :create, :roles => :db, :only => {:primary => true} do
+        file = with_additional_tags('backup') do
+          remote.create
+        end
+        if file.present?
+          DumpRake::Env.with_clean_env(:like => file) do
+            remote.download
+          end
+        end
+      end
+
+      desc "Uploads dump with backup tag and restores it on remote" << DumpRake::Env.explain_variables_for_command(:backup_restore)
+      task :restore, :roles => :db, :only => {:primary => true} do
+        file = with_additional_tags('backup') do
+          last_part_of_last_line(run_local(dump_command(:versions)))
+        end
+        if file.present?
+          DumpRake::Env.with_clean_env(:like => file) do
+            local.upload
+            remote.restore
+          end
+        end
+      end
+    end
+  end
+
+  after 'deploy:update_code' do
+    from, to = %W[#{shared_path}/dump #{release_path}/dump]
+    run "mkdir -p #{from}; rm -rf #{to}; ln -s #{from} #{to}"
+  end
+end

data/script/update_readme

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby -rrubygems
+
+$: << File.join(File.dirname(__FILE__), '../lib')
+require 'pathname'
+require 'dump_rake/env'
+
+readme = Pathname('README.markdown')
+lines = readme.readlines.map(&:rstrip)
+readme.open('w') do |f|
+  lines.each do |line|
+    line.sub!(/^`(.+?)`.*—.*$/) do
+      key, names = DumpRake::Env::DICTIONARY.find{ |key, values| values.include?($1)  }
+      if key
+        names = names.map{ |name| "`#{name}`" }.join(', ')
+        explanation = DumpRake::Env::EXPLANATIONS[key]
+        "#{names} — #{explanation}"
+      end
+    end
+    f.puts line
+  end
+end

data/spec/.gitignore

@@ -0,0 +1 @@
+/db/database.yml

data/spec/.tmignore

@@ -0,0 +1 @@
+/dummy-*

data/spec/cycle_spec.rb

@@ -0,0 +1,229 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+require File.dirname(__FILE__) + '/../lib/dump_rake'
+
+def database_configs
+  YAML::load(IO.read(PLUGIN_SPEC_DIR + "/db/database.yml"))
+end
+
+def adapters
+  database_configs.keys
+end
+
+def use_adapter(adapter)
+  config = database_configs[adapter]
+  begin
+    case config['adapter']
+    when /^mysql/
+      ActiveRecord::Base.establish_connection(config.merge('database' => nil))
+      ActiveRecord::Base.connection.create_database(config['database'])
+      ActiveRecord::Base.establish_connection(config)
+    when /^postgresql/
+      ActiveRecord::Base.establish_connection(config.merge('database' => 'postgres', 'schema_search_path' => 'public'))
+      ActiveRecord::Base.connection.create_database(config['database'])
+      ActiveRecord::Base.establish_connection(config)
+    else
+      ActiveRecord::Base.establish_connection(config)
+    end
+    load_schema
+    yield
+  ensure
+    case config['adapter']
+    when /^mysql/
+      ActiveRecord::Base.establish_connection(config)
+      ActiveRecord::Base.connection.drop_database config['database']
+    when /^postgresql/
+      ActiveRecord::Base.establish_connection(config.merge('database' => 'postgres', 'schema_search_path' => 'public'))
+      ActiveRecord::Base.connection.drop_database config['database']
+    end
+  end
+ensure
+  ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
+end
+
+def load_schema
+  grab_output{
+    load(DUMMY_SCHEMA_PATH)
+  }
+end
+
+def in_temp_rails_app
+  old_rails_root = DumpRake::RailsRoot.dup
+  DumpRake::RailsRoot.replace(File.join(PLUGIN_SPEC_DIR, 'temp_rails_app'))
+  FileUtils.remove_entry(DumpRake::RailsRoot) if File.exist?(DumpRake::RailsRoot)
+  FileUtils.mkpath(DumpRake::RailsRoot)
+  Progress.stub!(:io).and_return(StringIO.new)
+  yield
+ensure
+  FileUtils.remove_entry(DumpRake::RailsRoot) if File.exist?(DumpRake::RailsRoot)
+  DumpRake::RailsRoot.replace(old_rails_root)
+end
+
+def create_chickens!(options = {})
+  time = Time.local(2000,"jan",1,20,15,1)
+  data = {
+    :string => ['', 'lala'],
+    :text => ['', 'lala', 'lala' * 100],
+    :integer => [-1000, 0, 1000],
+    :float => [-1000.0, 0.0, 1000.0],
+    :decimal => [-1000.0, 0.0, 1000.0],
+    :datetime => [time, time - 5.years],
+    :timestamp => [time, time - 5.years],
+    :time => [time, time - 5.years],
+    :date => [time, time - 5.years],
+    :boolean => [true, false],
+  }
+  Chicken.create!
+  data.each do |type, values|
+    values.each do |value|
+      Chicken.create!("#{type}_col" => value)
+    end
+  end
+  if options[:random]
+    options[:random].to_i.times do
+      attributes = {}
+      data.each do |type, values|
+        attributes["#{type}_col"] = values[rand(values.length)] if rand > 0.5
+      end
+      Chicken.create!(attributes)
+    end
+  end
+end
+
+def reset_rake!
+  @rake = Rake::Application.new
+  Rake.application = @rake
+  load File.dirname(__FILE__) + '/../lib/tasks/assets.rake'
+  load File.dirname(__FILE__) + '/../lib/tasks/dump.rake'
+  Rake::Task.define_task('environment')
+  Rake::Task.define_task('db:schema:dump') do
+    File.open(DUMMY_SCHEMA_PATH, 'r') do |r|
+      if ENV['SCHEMA']
+        File.open(ENV['SCHEMA'], 'w') do |w|
+          w.write(r.read)
+        end
+      end
+    end
+  end
+  Rake::Task.define_task('db:schema:load') do
+    load(ENV['SCHEMA'])
+  end
+end
+
+def call_rake
+  reset_rake!
+  grab_output{
+    yield
+  }
+end
+
+def call_rake_create(*args)
+  call_rake{
+    DumpRake.create(*args)
+  }
+end
+
+def call_rake_restore(*args)
+  call_rake{
+    DumpRake.restore(*args)
+  }
+end
+
+describe 'full cycle' do
+  begin
+    database_configs
+
+    adapters.each do |adapter|
+      it "should dump and restore using #{adapter}" do
+        in_temp_rails_app do
+          use_adapter(adapter) do
+            #add chickens store their attributes and create dump
+            create_chickens!(:random => 100)
+            chicken_attributes = Chicken.all.map(&:attributes)
+            call_rake_create(:description => 'chickens')
+
+            #clear database
+            load_schema
+            Chicken.all.should == []
+
+            #restore dump and verify equality
+            call_rake_restore(:version => 'chickens')
+            Chicken.all.map(&:attributes).should == chicken_attributes
+
+            # go throught create/restore cycle and verify equality
+            call_rake_create
+            load_schema
+            Chicken.all.should be_empty
+            call_rake_restore
+            Chicken.all.map(&:attributes).should == chicken_attributes
+          end
+        end
+      end
+    end
+
+    adapters.each do |adapter|
+      it "should not break id incrementing using #{adapter}" do
+        in_temp_rails_app do
+          use_adapter(adapter) do
+            create_chickens!(:random => 100)
+            call_rake_create(:description => 'chickens')
+            load_schema
+            call_rake_restore(:version => 'chickens')
+            create_chickens!
+          end
+        end
+      end
+    end
+
+    adapters.combination(2) do |adapter_src, adapter_dst|
+      it "should dump using #{adapter_src} and restore using #{adapter_dst}" do
+        in_temp_rails_app do
+          chicken_attributes = nil
+          use_adapter(adapter_src) do
+            Chicken.all.should be_empty
+
+            create_chickens!(:random => 100)
+            chicken_attributes = Chicken.all.map(&:attributes)
+            call_rake_create
+          end
+
+          use_adapter(adapter_dst) do
+            Chicken.all.should be_empty
+
+            call_rake_restore
+            chicken_attributes.should == Chicken.all.map(&:attributes)
+          end
+        end
+      end
+    end
+
+    it "should create same dump for all adapters" do
+      in_temp_rails_app do
+        dumps = []
+        adapters.each do |adapter|
+          use_adapter(adapter) do
+            dump_name = call_rake_create(:desc => adapter)[:stdout].strip
+            dump_path = File.join(DumpRake::RailsRoot, 'dump', dump_name)
+
+            data = []
+            Zlib::GzipReader.open(dump_path) do |gzip|
+              Archive::Tar::Minitar.open(gzip, 'r') do |stream|
+                stream.each do |entry|
+                  data << [entry.full_name, entry.read]
+                end
+              end
+            end
+            dumps << {:path => dump_path, :data => data.sort}
+          end
+        end
+
+        dumps.combination(2) do |dump_a, dump_b|
+          dump_a[:path].should_not == dump_b[:path]
+          dump_a[:data].should == dump_b[:data]
+        end
+      end
+    end
+  rescue Errno::ENOENT => e
+    $stderr.puts e
+  end
+end