dump 1.0.0

data/lib/dump_rake/dump_writer.rb

@@ -0,0 +1,119 @@
+class DumpRake
+  class DumpWriter < Dump
+    attr_reader :stream, :config
+
+    def self.create(path)
+      new(path).open do |dump|
+        ActiveRecord::Base.logger.silence do
+          dump.write_schema
+
+          dump.write_tables
+          dump.write_assets
+
+          dump.write_config
+        end
+      end
+    end
+
+    def open
+      Pathname.new(path).dirname.mkpath
+      Zlib::GzipWriter.open(path) do |gzip|
+        gzip.mtime = Time.utc(2000)
+        lock do
+          Archive::Tar::Minitar.open(gzip, 'w') do |stream|
+            @stream = stream
+            @config = {:tables => {}}
+            yield(self)
+          end
+        end
+      end
+    end
+
+    def create_file(name)
+      Tempfile.open('dump') do |temp|
+        yield(temp)
+        temp.open
+        stream.tar.add_file_simple(name, :mode => 0100444, :size => temp.length) do |f|
+          f.write(temp.read(4096)) until temp.eof?
+        end
+      end
+    end
+
+    def write_schema
+      create_file('schema.rb') do |f|
+        DumpRake::Env.with_env('SCHEMA' => f.path) do
+          Rake::Task['db:schema:dump'].invoke
+        end
+      end
+    end
+
+    def write_tables
+      verify_connection
+      tables_to_dump.with_progress('Tables') do |table|
+        write_table(table)
+      end
+    end
+
+    def write_table(table)
+      row_count = table_row_count(table)
+      config[:tables][table] = row_count
+      Progress.start(table, 1 + row_count) do
+        create_file("#{table}.dump") do |f|
+          columns = table_columns(table)
+          column_names = columns.map(&:name).sort
+          columns_by_name = columns.index_by(&:name)
+
+          Marshal.dump(column_names, f)
+          Progress.step
+
+          each_table_row(table, row_count) do |row|
+            values = column_names.map do |column|
+              columns_by_name[column].type_cast(row[column])
+            end
+            Marshal.dump(values, f)
+            Progress.step
+          end
+        end
+      end
+    end
+
+    def write_assets
+      assets = assets_to_dump
+      if assets.present?
+        config[:assets] = {}
+        Dir.chdir(DumpRake::RailsRoot) do
+          assets = Dir[*assets].uniq
+          assets.with_progress('Assets') do |asset|
+            paths = Dir[File.join(asset, '**/*')]
+            files = paths.select{ |path| File.file?(path) }
+            config[:assets][asset] = {:total => paths.length, :files => files.length}
+            assets_root_link do |tmpdir, prefix|
+              paths.with_progress(asset) do |entry|
+                begin
+                  Archive::Tar::Minitar.pack_file(File.join(prefix, entry), stream)
+                rescue => e
+                  $stderr.puts "Skipped asset due to error #{e}"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def write_config
+      create_file('config') do |f|
+        Marshal.dump(config, f)
+      end
+    end
+
+    def assets_to_dump
+      begin
+        Rake::Task['assets'].invoke
+        DumpRake::Env[:assets].split(DumpRake::Assets::SPLITTER)
+      rescue
+        []
+      end
+    end
+  end
+end

data/lib/dump_rake/env.rb

@@ -0,0 +1,139 @@
+# encoding: UTF-8
+
+require 'dump_rake/env/filter'
+
+class DumpRake
+  module Env
+    DICTIONARY = {
+      :desc => %w[DESC DESCRIPTION],
+      :like => %w[LIKE VER VERSION],
+      :tags => %w[TAGS TAG],
+      :leave => %w[LEAVE],
+      :summary => %w[SUMMARY],
+      :assets => %w[ASSETS],
+      :tables => %w[TABLES],
+      :backup => %w[BACKUP AUTOBACKUP AUTO_BACKUP],
+      :transfer_via => %w[TRANSFER_VIA],
+      :migrate_down => %w[MIGRATE_DOWN],
+      :restore_schema => %w[RESTORE_SCHEMA],
+      :restore_tables => %w[RESTORE_TABLES],
+      :restore_assets => %w[RESTORE_ASSETS],
+      :show_size => %w[SHOW_SIZE], # internal
+    }.freeze unless defined? DICTIONARY
+
+    EXPLANATIONS = {
+      :desc => 'free form description of dump',
+      :like => 'filter dumps by full dump name',
+      :tags => 'comma separated list of tags',
+      :leave => 'number of dumps to leave',
+      :summary => 'output info about dump: "1", "true" or "yes" for basic info, "2" or "schema" to display schema as well',
+      :assets => 'comma or colon separated list of paths or globs to dump',
+      :tables => 'comma separated list of tables to dump or if prefixed by "-" — to skip; by default only sessions table is skipped; schema_info and schema_migrations are always included if they are present',
+      :backup => 'no autobackup if you pass "0", "no" or "false"',
+      :transfer_via => 'transfer method (rsync, sftp or scp)',
+      :migrate_down => 'don\'t run down for migrations not present in dump if you pass "0", "no" or "false"; pass "reset" to recreate (drop and create) db',
+      :restore_schema => 'don\'t read/change schema if you pass "0", "no" or "false" (useful to just restore data for table; note that schema info tables are also not restored)',
+      :restore_tables => 'works as TABLES, but for restoring',
+      :restore_assets => 'works as ASSETS, but for restoring',
+    }.freeze unless defined? EXPLANATIONS
+
+    class << self
+      def with_env(hash)
+        old = {}
+        hash.each do |key, value|
+          key = DICTIONARY[key].first if DICTIONARY[key]
+          old[key] = ENV[key]
+          ENV[key] = value
+        end
+        begin
+          yield
+        ensure
+          old.each do |key, value|
+            ENV[key] = value
+          end
+        end
+      end
+
+      def with_clean_env(hash = {}, &block)
+        empty_env = {}
+        DICTIONARY.keys.each{ |key| empty_env[key] = nil }
+        with_env(empty_env.merge(hash), &block)
+      end
+
+      def [](key)
+        if DICTIONARY[key]
+          ENV.values_at(*DICTIONARY[key]).compact.first
+        else
+          ENV[key]
+        end
+      end
+
+      def filter(key, splitter = nil)
+        @filters ||= Hash.new{ |hash, key| hash[key] = Filter.new(*key) }
+        @filters[[self[key], splitter]]
+      end
+
+      def yes?(key)
+        %w[1 y t].include?(first_char(key))
+      end
+
+      def no?(key)
+        %w[0 n f].include?(first_char(key))
+      end
+
+      def downcase(key)
+        self[key].to_s.downcase.strip
+      end
+
+      def variable_names_for_command(command)
+        m = {
+          :select => [:like, :tags],
+          :assets => [:assets],
+          :restore_options => [:migrate_down, :restore_schema, :restore_tables, :restore_assets],
+          :transfer_options => [:transfer_via]
+        }
+
+        m[:versions] = m[:select] | [:summary]
+        m[:create] = [:desc, :tags, :tables] | m[:assets]
+        m[:restore] = m[:select] | m[:restore_options]
+        m[:cleanup] = m[:select] | [:leave]
+
+        m[:transfer] = m[:select] | m[:transfer_options]
+
+        m[:mirror] = [:backup] | m[:create] | m[:transfer_options] | m[:restore_options]
+        m[:backup] = m[:create] | [:transfer_via]
+        m[:backup_restore] = m[:transfer] | m[:restore_options]
+
+        m[command] || []
+      end
+
+      def for_command(command, strings = false)
+        variables = variable_names_for_command(command)
+        variables.inject({}) do |env, variable|
+          value = self[variable]
+          env[strings ? DICTIONARY[variable].first : variable] = value if value
+          env
+        end
+      end
+
+      def stringify!(hash)
+        hash.keys.each do |key|
+          hash[DICTIONARY[key] ? DICTIONARY[key].first : key.to_s] = hash.delete(key)
+        end
+      end
+
+      def explain_variables_for_command(command)
+        ".\n" <<
+        variable_names_for_command(command).map do |variable_name|
+          "  #{DICTIONARY[variable_name].join(', ')} — #{EXPLANATIONS[variable_name]}\n"
+        end.join('')
+      end
+
+    private
+
+      def first_char(key)
+        downcase(key)[0, 1]
+      end
+    end
+  end
+end

data/lib/dump_rake/env/filter.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+class DumpRake
+  module Env
+    class Filter
+      attr_reader :invert, :values, :transparent
+      def initialize(s, splitter = nil)
+        if s
+          s = s.dup
+          @invert = !!s.sub!(/^-/, '')
+          @values = s.split(splitter || ',').map(&:strip).map(&:downcase).uniq.select(&:present?)
+        else
+          @transparent = true
+        end
+      end
+
+      def pass?(value)
+        transparent || (invert ^ values.include?(value.to_s.downcase))
+      end
+
+      def custom_pass?(&block)
+        transparent || (invert ^ values.any?(&block))
+      end
+    end
+  end
+end

data/lib/dump_rake/rails_root.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+class DumpRake
+  RailsRoot = case
+  when defined?(Rails)
+    Rails.root
+  when defined?(RAILS_ROOT)
+    RAILS_ROOT
+  else
+    Dir.pwd
+  end.to_s
+end

data/lib/dump_rake/table_manipulation.rb

@@ -0,0 +1,131 @@
+class DumpRake
+  module TableManipulation
+  protected
+
+    def schema_tables
+      %w[schema_info schema_migrations]
+    end
+
+    def verify_connection
+      ActiveRecord::Base.connection.verify!(0)
+    end
+
+
+    def quote_table_name(table)
+      ActiveRecord::Base.connection.quote_table_name(table)
+    end
+
+    def quote_column_name(column)
+      ActiveRecord::Base.connection.quote_column_name(column)
+    end
+
+    def quote_value(value)
+      ActiveRecord::Base.connection.quote(value)
+    end
+
+
+    def clear_table(table_sql)
+      ActiveRecord::Base.connection.delete("DELETE FROM #{table_sql}", 'Clearing table')
+    end
+
+    def insert_into_table(table_sql, columns_sql, values_sql)
+      values_sql = values_sql.join(',') if values_sql.is_a?(Array)
+      ActiveRecord::Base.connection.insert("INSERT INTO #{table_sql} #{columns_sql} VALUES #{values_sql}", 'Loading dump')
+    end
+
+    def fix_sequence!(table)
+      if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
+        ActiveRecord::Base.connection.reset_pk_sequence!(table)
+      end
+    end
+
+    def join_for_sql(quoted)
+      "(#{quoted.join(',')})"
+    end
+
+    def columns_insert_sql(columns)
+      join_for_sql(columns.map{ |column| quote_column_name(column) })
+    end
+
+    def values_insert_sql(values)
+      join_for_sql(values.map{ |value| quote_value(value) })
+    end
+
+
+    def avaliable_tables
+      ActiveRecord::Base.connection.tables
+    end
+
+    def tables_to_dump
+      if DumpRake::Env[:tables]
+        avaliable_tables.select do |table|
+          schema_tables.include?(table) || DumpRake::Env.filter(:tables).pass?(table)
+        end
+      else
+        avaliable_tables - %w[sessions]
+      end
+    end
+
+    def table_row_count(table)
+      ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM #{quote_table_name(table)}").to_i
+    end
+
+    CHUNK_SIZE_MIN = 100 unless const_defined?(:CHUNK_SIZE_MIN)
+    CHUNK_SIZE_MAX = 3_000 unless const_defined?(:CHUNK_SIZE_MAX)
+    def table_chunk_size(table)
+      expected_row_size = table_columns(table).map do |column|
+        case column.type
+        when :text
+          Math.sqrt(column.limit || 2_147_483_647)
+        when :string
+          Math.sqrt(column.limit || 255)
+        else
+          column.limit || 10
+        end
+      end.sum
+      [[(10_000_000 / expected_row_size).round, CHUNK_SIZE_MIN].max, CHUNK_SIZE_MAX].min
+    end
+
+    def table_columns(table)
+      ActiveRecord::Base.connection.columns(table)
+    end
+
+    def table_has_primary_column?(table)
+      # bad test for primary column, but primary even for primary column is nil
+      table_columns(table).any?{ |column| column.name == table_primary_key(table) && column.type == :integer }
+    end
+
+    def table_primary_key(table)
+      'id'
+    end
+
+    def each_table_row(table, row_count, &block)
+      if table_has_primary_column?(table) && row_count > (chunk_size = table_chunk_size(table))
+        # adapted from ActiveRecord::Batches
+        primary_key = table_primary_key(table)
+        quoted_primary_key = "#{quote_table_name(table)}.#{quote_column_name(primary_key)}"
+        select_where_primary_key =
+          "SELECT * FROM #{quote_table_name(table)}" +
+            " WHERE #{quoted_primary_key} %s" +
+            " ORDER BY #{quoted_primary_key} ASC" +
+            " LIMIT #{chunk_size}"
+        rows = select_all_by_sql(select_where_primary_key % '>= 0')
+        until rows.blank?
+          rows.each(&block)
+          break if rows.length < chunk_size
+          rows = select_all_by_sql(select_where_primary_key % "> #{rows.last[primary_key].to_i}")
+        end
+      else
+        table_rows(table).each(&block)
+      end
+    end
+
+    def table_rows(table)
+      select_all_by_sql("SELECT * FROM #{quote_table_name(table)}")
+    end
+
+    def select_all_by_sql(sql)
+      ActiveRecord::Base.connection.select_all(sql)
+    end
+  end
+end

data/lib/generators/assets_config/assets_config_generator.rb

@@ -0,0 +1,16 @@
+case
+when defined?(Rails::Generator::Base)
+  class AssetsConfigGenerator < Rails::Generator::Base
+    def manifest
+      record do |m|
+        m.file 'assets', 'config/assets'
+      end
+    end
+  end
+when defined?(Rails::Generators::Base)
+  class AssetsConfigGenerator < Rails::Generators::Base
+    def create_assets_config
+      create_file 'config/assets', File.read(File.join(File.dirname(__FILE__), 'templates/assets'))
+    end
+  end
+end