dump 1.0.0

data/lib/dump_rake/continious_timeout.rb

@@ -0,0 +1,38 @@
+# based on Timeout
+
+module ContiniousTimeout
+  class TimeoutException < ::Exception
+  end
+
+  class RestartException < ::Exception
+  end
+
+  class Deferer
+    def initialize(thread)
+      @thread = thread
+    end
+
+    def defer
+      @thread.raise RestartException.new
+    end
+  end
+
+  def self.timeout(sec)
+    begin
+      x = Thread.current
+      y = Thread.start do
+        1.times do
+          begin
+            sleep sec
+          rescue RestartException => e
+            retry
+          end
+        end
+        x.raise TimeoutException, "execution expired" if x.alive?
+      end
+      yield Deferer.new(y)
+    ensure
+      y.kill if y and y.alive?
+    end
+  end
+end

data/lib/dump_rake/dump.rb

@@ -0,0 +1,175 @@
+# encoding: UTF-8
+
+class DumpRake
+  class Dump
+    include TableManipulation
+    def self.list(options = {})
+      dumps = Dir[File.join(DumpRake::RailsRoot, 'dump', options[:all] ? '*.*' : '*.tgz')].sort.select{ |path| File.file?(path) }.map{ |path| new(path) }
+      dumps = dumps.select{ |dump| dump.name[options[:like]] } if options[:like]
+      if options[:tags]
+        tags = get_filter_tags(options[:tags])
+        dumps = dumps.select{ |dump| (dump.tags & tags[:simple]).present? } if tags[:simple].present?
+        dumps = dumps.select{ |dump| (dump.tags & tags[:mandatory]) == tags[:mandatory] } if tags[:mandatory].present?
+        dumps = dumps.reject{ |dump| (dump.tags & tags[:forbidden]).present? } if tags[:forbidden].present?
+      end
+      dumps
+    end
+
+    def initialize(path_or_options = {})
+      if path_or_options.is_a?(Hash)
+        options = path_or_options
+
+        name = Time.now.utc.strftime("%Y%m%d%H%M%S")
+
+        description = clean_description(options[:desc])
+        name += "-#{description}" unless description.blank?
+
+        tags = clean_tags(options[:tags])
+        name += "@#{tags * ','}" unless tags.empty?
+
+        tgz_name = "#{name}.tgz"
+
+        @path = options[:dir] ? Pathname(options[:dir]) + tgz_name : Pathname(tgz_name)
+
+      else
+        @path = Pathname(path_or_options)
+      end
+    end
+
+    attr_reader :path
+
+    def tgz_path
+      path_with_ext('tgz')
+    end
+
+    def tmp_path
+      path_with_ext('tmp')
+    end
+
+    def ==(other)
+      path == other.path
+    end
+
+    def parts
+      @parts ||=
+      if m = name.match(/^(\d{#{4+2+2 + 2+2+2}})(-[^@]+)?((?:@[^@]+)+)?\.(tmp|tgz)$/)
+        {
+          :time => m[1],
+          :desc => m[2] && m[2][1, m[2].length],
+          :tags => m[3] && m[3][1, m[3].length],
+          :ext => m[4]
+        }
+      else
+        {}
+      end
+    end
+
+    def time
+      parts[:time] && Time.utc(*parts[:time].match(/(\d{4})#{'(\d{2})' * 5}/)[1..6])
+    end
+
+    def description
+      clean_description(parts[:desc])
+    end
+
+    def tags
+      clean_tags(parts[:tags])
+    end
+
+    def ext
+      parts[:ext]
+    end
+
+    def name
+      @name ||= File.basename(path)
+    end
+    alias to_s name
+
+    def size
+      File.size(path) rescue nil
+    end
+
+    def human_size
+      number = size
+      return nil if number.nil?
+      degree = 0
+      symbols = %W[B K M G T]
+      while number >= 1000 && degree < symbols.length - 1
+        degree += 1
+        number /= 1024.0
+      end
+      "#{'%.2f' % number}#{symbols[degree]}"
+    end
+
+    def inspect
+      "#<%s:0x%x %s>" % [self.class, object_id, path.to_s.sub(/^.+(?=..\/[^\/]*$)/, '…')]
+    end
+
+    def lock
+      if lock = File.open(path, 'r')
+        begin
+          if lock.flock(File::LOCK_EX | File::LOCK_NB)
+            yield
+          end
+        ensure
+          lock.flock(File::LOCK_UN)
+          lock.close
+        end
+      end
+    end
+
+  protected
+
+    def assets_root_link
+      prefix = 'assets'
+      Dir.mktmpdir do |dir|
+        Dir.chdir(dir) do
+          File.symlink(DumpRake::RailsRoot, prefix)
+          begin
+            yield dir, prefix
+          ensure
+            File.unlink(prefix)
+          end
+        end
+      end
+    end
+
+    def path_with_ext(ext)
+      Pathname(path.to_s.sub(/#{parts[:ext]}$/, ext))
+    end
+
+    module CleanNParse
+      def clean_str(str, additional = nil)
+        str.to_s.strip.gsub(/\s+/, ' ').gsub(/[^A-Za-z0-9 \-_#{Regexp.escape(additional.to_s) if additional}]+/, '_')
+      end
+      def clean_description(description)
+        clean_str(description, '()#')[0, 50].strip
+      end
+      def clean_tag(tag)
+        clean_str(tag).downcase.sub(/^\-+/, '')[0, 20].strip
+      end
+      def clean_tags(tags)
+        tags.to_s.split(',').map{ |tag| clean_tag(tag) }.uniq.reject(&:blank?).sort
+      end
+      def get_filter_tags(tags)
+        groups = Hash.new{ |hash, key| hash[key] = SortedSet.new }
+        tags.to_s.split(',').each do |tag|
+          if m = tag.strip.match(/^(\-|\+)?(.*)$/)
+            type = {'+' => :mandatory, '-' => :forbidden}[m[1]] || :simple
+            unless (claned_tag = clean_tag(m[2])).blank?
+              groups[type] << claned_tag
+            end
+          end
+        end
+        [:simple, :mandatory].each do |type|
+          if (clashing = (groups[type] & groups[:forbidden])).present?
+            raise "#{type} tags clashes with forbidden ones: #{clashing}"
+          end
+        end
+        groups.each_with_object({}){ |(key, value), hsh| hsh[key] = value.to_a }
+      end
+    end
+    include CleanNParse
+    extend CleanNParse
+  end
+end

data/lib/dump_rake/dump_reader.rb

@@ -0,0 +1,289 @@
+class DumpRake
+  class DumpReader < Dump
+    attr_reader :stream, :config
+
+    def self.restore(path)
+      new(path).open do |dump|
+        ActiveRecord::Base.logger.silence do
+          dump.read_config
+          dump.migrate_down
+          dump.read_schema
+
+          dump.read_tables
+          dump.read_assets
+        end
+      end
+    end
+
+    class Summary
+      attr_reader :text
+      alias_method :to_s, :text
+      def initialize
+        @text = ''
+      end
+
+      def header(header)
+        @text << "  #{header}:\n"
+      end
+
+      def data(entries)
+        entries.each do |entry|
+          @text << "    #{entry}\n"
+        end
+      end
+
+      # from ActionView::Helpers::TextHelper
+      def self.pluralize(count, singular)
+        "#{count} #{count == 1 ? singular : singular.pluralize}"
+      end
+    end
+
+    def self.summary(path, options = {})
+      new(path).open do |dump|
+        dump.read_config
+
+        sum = Summary.new
+
+        tables = dump.config[:tables]
+        sum.header 'Tables'
+        sum.data tables.sort.map{ |(table, rows)|
+          "#{table}: #{Summary.pluralize(rows, 'row')}"
+        }
+
+        assets = dump.config[:assets]
+        if assets.present?
+          sum.header 'Assets'
+          sum.data assets.sort.map{ |entry|
+            if String === entry
+              entry
+            else
+              asset, paths = entry
+              if Hash === paths
+                "#{asset}: #{Summary.pluralize paths[:files], 'file'} (#{Summary.pluralize paths[:total], 'entry'} total)"
+              else
+                "#{asset}: #{Summary.pluralize paths, 'entry'}"
+              end
+            end
+          }
+        end
+
+        if options[:schema]
+          sum.header 'Schema'
+          sum.data dump.schema.split("\n")
+        end
+
+        sum
+      end
+    end
+
+    def open
+      Zlib::GzipReader.open(path) do |gzip|
+        Archive::Tar::Minitar.open(gzip, 'r') do |stream|
+          @stream = stream
+          yield(self)
+        end
+      end
+    end
+
+    def find_entry(matcher)
+      stream.each do |entry|
+        if matcher === entry.full_name
+          # we can not return entry - after exiting stream.each the entry will be invalid and will read from tar start
+          return yield(entry)
+        end
+      end
+    end
+
+    def read_entry(matcher)
+      find_entry(matcher) do |entry|
+        return entry.read
+      end
+    end
+
+    def read_entry_to_file(matcher)
+      find_entry(matcher) do |entry|
+        Tempfile.open('dumper') do |temp|
+          temp.write(entry.read(4096)) until entry.eof?
+          temp.rewind
+          yield(temp)
+        end
+      end
+    end
+
+    def read_config
+      @config = Marshal.load(read_entry('config'))
+    end
+
+    def migrate_down
+      case
+      when DumpRake::Env.downcase(:migrate_down) == 'reset'
+        Rake::Task['db:drop'].invoke
+        Rake::Task['db:create'].invoke
+      when !DumpRake::Env.no?(:migrate_down)
+        if avaliable_tables.include?('schema_migrations')
+          find_entry("schema_migrations.dump") do |entry|
+            migrated = table_rows('schema_migrations').map{ |row| row['version'] }
+
+            dump_migrations = []
+            Marshal.load(entry) # skip header
+            dump_migrations << Marshal.load(entry).first until entry.eof?
+
+            migrate_down = (migrated - dump_migrations)
+
+            unless migrate_down.empty?
+              migrate_down.reverse.with_progress('Migrating down') do |version|
+                DumpRake::Env.with_env('VERSION' => version) do
+                  Rake::Task['db:migrate:down'].tap do |task|
+                    begin
+                      task.invoke
+                    rescue ActiveRecord::IrreversibleMigration
+                      $stderr.puts "Irreversible migration: #{version}"
+                    end
+                    task.reenable
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def restore_schema?
+      !DumpRake::Env.no?(:restore_schema)
+    end
+
+    def read_schema
+      if restore_schema?
+        read_entry_to_file('schema.rb') do |f|
+          DumpRake::Env.with_env('SCHEMA' => f.path) do
+            Rake::Task['db:schema:load'].invoke
+          end
+          Rake::Task['db:schema:dump'].invoke
+        end
+      end
+    end
+
+    def schema
+      read_entry('schema.rb')
+    end
+
+    def read_tables
+      verify_connection
+      config[:tables].with_progress('Tables') do |table, rows|
+        if (restore_schema? && schema_tables.include?(table)) || DumpRake::Env.filter(:restore_tables).pass?(table)
+          read_table(table, rows)
+        end
+      end
+    end
+
+    def read_table(table, rows_count)
+      find_entry("#{table}.dump") do |entry|
+        table_sql = quote_table_name(table)
+        clear_table(table_sql)
+
+        columns = Marshal.load(entry)
+        columns_sql = columns_insert_sql(columns)
+        Progress.start(table, rows_count) do
+          until entry.eof?
+            rows_sql = []
+            1000.times do
+              rows_sql << values_insert_sql(Marshal.load(entry)) unless entry.eof?
+            end
+
+            begin
+              insert_into_table(table_sql, columns_sql, rows_sql)
+              Progress.step(rows_sql.length)
+            rescue
+              rows_sql.each do |row_sql|
+                insert_into_table(table_sql, columns_sql, row_sql)
+                Progress.step
+              end
+            end
+          end
+        end
+        fix_sequence!(table)
+      end
+    end
+
+    def read_assets
+      unless config[:assets].blank?
+        assets = config[:assets]
+        if Hash === assets
+          assets_count = assets.values.sum{ |value| Hash === value ? value[:total] : value }
+          assets_paths = assets.keys
+        else
+          assets_count, assets_paths = nil, assets
+        end
+
+        if DumpRake::Env[:restore_assets]
+          assets_paths.each do |asset|
+            DumpRake::Assets.glob_asset_children(asset, '**/*').reverse.each do |child|
+              if read_asset?(child, DumpRake::RailsRoot)
+                case
+                when File.file?(child)
+                  File.unlink(child)
+                when File.directory?(child)
+                  begin
+                    Dir.unlink(child)
+                  rescue Errno::ENOTEMPTY
+                    nil
+                  end
+                end
+              end
+            end
+          end
+        else
+          DumpRake::Env.with_env(:assets => assets_paths.join(':')) do
+            Rake::Task['assets:delete'].invoke
+          end
+        end
+
+        read_assets_entries(assets_paths, assets_count) do |stream, root, entry, prefix|
+          if !DumpRake::Env[:restore_assets] || read_asset?(entry.full_name, prefix)
+            stream.extract_entry(root, entry)
+          end
+        end
+      end
+    end
+
+    def read_asset?(path, prefix)
+      DumpRake::Env.filter(:restore_assets, DumpRake::Assets::SPLITTER).custom_pass? do |value|
+        File.fnmatch(File.join(prefix, value), path) ||
+        File.fnmatch(File.join(prefix, value, '**'), path)
+      end
+    end
+
+    def read_assets_entries(assets_paths, assets_count)
+      Progress.start('Assets', assets_count || 1) do
+        found_assets = false
+        # old style — in separate tar
+        find_entry('assets.tar') do |assets_tar|
+          def assets_tar.rewind
+            # rewind will fail - it must go to center of gzip
+            # also we don't need it - this is last step in dump restore
+          end
+          Archive::Tar::Minitar.open(assets_tar) do |inp|
+            inp.each do |entry|
+              yield inp, DumpRake::RailsRoot, entry, nil
+              Progress.step if assets_count
+            end
+          end
+          found_assets = true
+        end
+
+        unless found_assets
+          # new style — in same tar
+          assets_root_link do |tmpdir, prefix|
+            stream.each do |entry|
+              if entry.full_name.starts_with?("#{prefix}/")
+                yield stream, tmpdir, entry, prefix
+                Progress.step if assets_count
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end