tiny_backup 0.0.0

data/lib/tiny_backup.rb

@@ -0,0 +1,613 @@
+begin
+  # there is a problem between two versions of the zip gem
+  require 'zip'
+  ZIPLIB = Zip::File
+  ZIPOLD = false
+rescue LoadError
+  ZIPOLD = true
+  ZIPLIB = Zip::ZipFile
+end
+require 'rake'
+require 'csv'
+
+module TinyBackup
+  # Get configuration settings
+  #
+  # Returns a *hash* with the following keys backup_folder, date_format, version_prefix, zip_prefix, max_versions, per_page
+  def config
+    {
+      backup_folder: backup_folder,
+      date_format: date_format,
+      version_prefix: version_prefix,
+      zip_prefix: zip_prefix,
+      max_versions: max_versions,
+      per_page: per_page
+    }
+  end
+
+  # Set configuration settings
+  #
+  # +options+ is a *hash* with the following keys backup_folder, date_format, version_prefix, zip_prefix, max_versions, per_page
+  def config= options
+    if options[:date_format].present?
+      # fall-back if the date_format is invalid
+      begin
+        Time.now.strftime(options[:date_format]).to_datetime
+      rescue
+        raise ArgumentError, "invalid date format"
+      end
+    end
+
+    options.slice(*config.keys).each do |k, v|
+      instance_variable_set "@#{k}", ([:max_versions, :per_page].include?(k) ? v.to_i : v.to_s)
+    end
+  end
+
+  # Returns a *string* of the folder that is used to keep the backup files and temporary files
+  def backup_folder
+    this_backup_folder = @backup_folder || "db/backup"
+
+    # the backup_folder will not have any slash at the end
+    this_backup_folder = this_backup_folder.to_s[0..-2] if this_backup_folder.to_s.last == "/"
+
+    # create the folder if doesn't exist
+    FileUtils.mkdir(this_backup_folder) unless File.directory?(this_backup_folder)
+
+    return this_backup_folder
+  end
+
+  # Returns a *string* of the date format that will be appended and used to time-stamp the backup files. Can be any valid date format similar with the ISO C and POSIX directives
+  #
+  # *Default* *value*: %d-%m-%Y_%H:%M
+  def date_format
+    @date_format || "%d-%m-%Y_%H:%M"
+  end
+
+  # Returns a *string* of the prefix used to name each .diff file that contains changes from the last version. Automatically generated +version_number+ will be appended to the +version_prefix+
+  #
+  # *Default* *value*: +ver_+
+  def version_prefix
+    @version_prefix || "ver_"
+  end
+
+  # Returns a *string* of the prefix used to name the starting .zip file that contains a schema.rb file and a .csv file for each non-empty table
+  #
+  # *Default* *value*: +origin+
+  def zip_prefix
+    @zip_prefix || "origin"
+  end
+
+  # Returns an *integer* of maximum number of versions that can exist. If this number is exceeded, the first .diff version is merged in the starting .zip file and cannot be reversed
+  #
+  # *Default* *value*: +32+ (if backup is done daily, keep the last month versions)
+  def max_versions
+    @max_versions || 32
+  end
+
+  # Returns an *integer* of maximum number of rows fetched by a single query to avoid memory crashes. If your table has more rows it will be used multiple query to fetch all the data
+  #
+  # *Default* *value*: +100000+
+  def per_page
+    @per_page || 100000
+  end
+
+  # Create a backup of the current database and choose automatically to create a .zip or .diff file
+  #
+  # *Lock* *until* *is* *done*
+  def backup_now
+    # if the resource is locked, we skip to ensure block
+    lock
+    locked_by_this_method = true
+
+    @tmp_files = []
+    nvf = new_version_filename
+
+    stream = StringIO.new
+    ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
+    schema_rb = stream.string
+
+    if nvf.split(".").last == "zip"
+      # there is no .zip file so we must create one and add schema.rb and .csv file for each table
+      # TODO: use a much better compression like Zlib::BEST_COMPRESSION to reduce the zip size, but this will consume processing power
+      ZIPLIB.open("#{backup_folder}/#{nvf}", ZIPLIB::CREATE) do |f|
+        puts "-- backup_schema\n"
+        t_benchmark = Benchmark.ms do
+          f.get_output_stream("schema.rb") do |ff|
+            ff.write schema_rb
+            @tmp_files << ff if ZIPOLD
+          end
+        end
+        puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+
+        ActiveRecord::Base.connection.tables.each do |table|
+          next if table == "schema_migrations"
+          query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first
+
+          puts "-- backup_table(\"#{table}\")\n"
+          t_benchmark = Benchmark.ms do
+            if query_count > 0
+              rows = []
+              query_index = 0
+
+              loop do
+                break if query_index >= query_count
+                query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{per_page} OFFSET #{query_index}")
+                rows << add_query(query.fields) if query_index == 0
+                query.each { |row| rows << add_query(row) }
+                query_index += per_page
+              end
+
+              f.get_output_stream("#{table}.csv") do |ff|
+                ff.write rows.join
+                @tmp_files << ff if ZIPOLD
+              end
+            end
+          end
+          puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+        end
+      end
+
+    else
+      # a new .diff file is created with the diff between origin_zip and tmp_origin_zip(made by merging all the versions into origin)
+      tmp_origin_zip = "#{backup_folder}/#{compact_original(:all)}"
+      tables = ActiveRecord::Base.connection.tables
+      is_empty = true
+
+      File.open("#{backup_folder}/#{nvf}", "wb") do |f|
+        ZIPLIB.open(tmp_origin_zip) do |zf|
+          zf.entries.each do |zf_entry|
+
+            if zf_entry.name == "schema.rb"
+              puts "-- backup_schema\n"
+              t_benchmark = Benchmark.ms do
+                tables.delete "schema_migrations"
+                this_diff = diff_files zf.read(zf_entry.name), schema_rb
+
+                if this_diff.present?
+                  is_empty = false
+                  f.write "***************\n"
+                  f.write "*** schema.rb \n"
+                  f.write "\n"
+                  this_diff.each { |i| f.write i }
+                  f.write "\n\n"
+                end
+              end
+              puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+            else
+
+              table = zf_entry.name.split(".").first
+              tables.delete table
+              begin
+                query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first
+              rescue ActiveRecord::StatementInvalid
+                next
+              end
+
+              rows = []
+              query_index = 0
+
+              puts "-- backup_table(\"#{table}\")\n"
+              t_benchmark = Benchmark.ms do
+                loop do
+                  break if query_index >= query_count
+                  query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{per_page} OFFSET #{query_index}")
+                  rows << add_query(query.fields) if query_index == 0
+                  query.each { |row| rows << add_query(row) }
+                  query_index += per_page
+                end
+
+                this_diff = diff_files zf.read(zf_entry.name), rows.join
+
+                if this_diff.present?
+                  is_empty = false
+                  f.write "***************\n"
+                  f.write "*** #{zf_entry.name} \n"
+                  f.write "\n"
+                  this_diff.each { |i| f.write i }
+                  f.write "\n\n"
+                end
+              end
+              puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+            end
+
+          end
+        end
+
+        # tables that are created recently and doesn't have a .csv file in the tmp_origin_zip
+        tables.each do |table|
+          begin
+            query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first
+          rescue ActiveRecord::StatementInvalid
+            next
+          end
+
+          rows = []
+          query_index = 0
+
+          puts "-- backup_table(\"#{table}\")\n"
+          t_benchmark = Benchmark.ms do
+            loop do
+              break if query_index >= query_count
+              query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{per_page} OFFSET #{query_index}")
+
+              rows << add_query(query.fields) if query_index == 0
+              query.each { |row| rows << add_query(row) }
+              query_index += per_page
+            end
+
+            this_diff = diff_files "", rows.join
+
+            if this_diff.present?
+              is_empty = false
+              f.write "***************\n"
+              f.write "*** #{table}.csv \n"
+              f.write "\n"
+              this_diff.each { |i| f.write i }
+              f.write "\n\n"
+            end
+          end
+          puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+        end
+      end
+
+      File.delete tmp_origin_zip
+      File.delete("#{backup_folder}/#{nvf}") if is_empty
+    end
+
+    # keep max versions
+    version_files = Dir.glob("#{backup_folder}/#{version_prefix}*").sort
+    if max_versions < version_files.length
+      # throw files to garbage
+      @tmp_files << Dir.glob("#{backup_folder}/#{zip_prefix}*").first
+      @tmp_files << version_files.first
+
+      File.rename "#{backup_folder}/#{compact_original(1)}", dup_file("#{backup_folder}/#{zip_prefix}_#{Time.now.strftime(date_format)}.zip")
+    end
+
+    # delete temporary files before method exit
+    rescue => e
+      @method_error = e
+    ensure
+      unlock if locked_by_this_method
+      @tmp_files.each { |i| File.delete(i) rescue nil } if @tmp_files.present?
+      raise @method_error if @method_error.present?
+      return true
+  end
+
+  # Merge into the .zip file and delete all the .diff files to clear the space
+  #
+  # *Lock* *until* *is* *done*
+  def compact_all
+    lock # if the resource is locked, we skip to ensure block
+    locked_by_this_method = true
+
+    @tmp_files = []
+    @tmp_files += Dir.glob("#{backup_folder}/#{zip_prefix}*")
+    @tmp_files += Dir.glob("#{backup_folder}/#{version_prefix}*")
+
+    # make the temporary zip be the original and apply the updated_at time-stamp
+    File.rename "#{backup_folder}/#{compact_original(:all)}", dup_file("#{backup_folder}/#{zip_prefix}_#{Time.now.strftime(date_format)}.zip")
+
+    # delete temporary files before method exit
+    rescue => e
+      @method_error = e
+    ensure
+      unlock if locked_by_this_method
+      @tmp_files.each { |i| File.delete(i) rescue nil } if @tmp_files.present?
+      raise @method_error if @method_error.present?
+      return true
+  end
+
+  # Change the database to match the selected integer +version_number+.
+  # If the +just_temporary+ is false, the unused version files will be deleted and the latest backup version will be synchronized with the database data.
+  # It's dangerous to start a new backup if the database is not synchronized with the latest backup version(call restore_db(:all) to restore all data)
+  #
+  # *Lock* *until* *is* *done*
+  def restore_db version_number, just_temporary=true
+    lock # if the resource is locked, we skip to ensure block
+    locked_by_this_method = true
+
+    @tmp_files = []
+
+    if just_temporary && version_number != :all
+      puts "you want to restore just temporary: DO NOT start a backup BEFORE calling TinyBackup.restore_db(:all)\n"
+    end
+
+    version_files = Dir.glob("#{backup_folder}/#{version_prefix}*")
+    if version_number == :all
+      version_count = version_files.length
+    else
+      good_versions = version_files.find_all { |i| i.gsub("#{backup_folder}/#{version_prefix}", "").split("_").first.to_i <= version_number.to_i }
+      version_count = good_versions.length
+      @tmp_files += version_files - good_versions if !just_temporary
+    end
+
+    tmp_origin_zip = compact_original version_count
+    @tmp_files << "#{backup_folder}/#{tmp_origin_zip}"
+    @tmp_files << "#{backup_folder}/schema_tmp.rb"
+
+    db_name = Rails.configuration.database_configuration[Rails.env]["database"]
+    db_collation = ActiveRecord::Base.connection.collation
+    ActiveRecord::Base.connection.drop_database   db_name
+    ActiveRecord::Base.connection.create_database db_name, collation: db_collation
+    ActiveRecord::Base.connection.reconnect!
+
+    # prepare the structure
+    ZIPLIB.open("#{backup_folder}/#{tmp_origin_zip}") do |zf|
+      zf.entries.each do |zf_entry|
+        if zf_entry.name == "schema.rb"
+          File.open("#{backup_folder}/schema_tmp.rb", "wb") { |f| f.write zf.read(zf_entry.name) }
+          break
+        end
+      end
+    end
+    ActiveRecord::Schema.load("#{backup_folder}/schema_tmp.rb")
+
+    # add the data
+    ZIPLIB.open("#{backup_folder}/#{tmp_origin_zip}") do |zf|
+      zf.entries.each do |zf_entry|
+        next if zf_entry.name == "schema.rb"
+        table_rows = zf.read(zf_entry.name).split("\n")
+        table_header = table_rows.shift
+        table_name = zf_entry.name.split(".").first
+
+        puts "-- insert_data(\"#{table_name}\")\n"
+        t_benchmark = Benchmark.ms do
+          table_rows.in_groups_of(per_page, false) do |tr_group|
+            ActiveRecord::Base.connection.execute insert_row(table_name, table_header, tr_group)
+          end
+        end
+        puts "   -> #{'%.4f' % (t_benchmark/1000)}s\n"
+      end
+    end
+
+    # delete temporary files before method exit
+    rescue => e
+      @method_error = e
+    ensure
+      unlock if locked_by_this_method
+      @tmp_files.each { |i| File.delete(i) rescue nil } if @tmp_files.present?
+      raise @method_error if @method_error.present?
+      return true
+  end
+
+  # YOU SHALL NOT PASS -----------------------------------------------------------------
+  private
+
+  # Returns a *string* of the query used to insert a row of data
+  def insert_row(table_name, table_header, table_rows)
+    "INSERT INTO #{table_name} (#{table_header.gsub("\"", "`")}) VALUES " +
+      table_rows.map { |i| "(#{i})" }.join(",")
+  end
+
+  # Merge X versions starting with the lowest +version_number+ into a temporary .zip file that looks like the original zip
+  #
+  # Returns a *string* that is the path of the temporary zip file
+  def compact_original versions
+    versions = Dir.glob("#{backup_folder}/*").length if versions == :all
+    @tmp_files_compact ||= []
+    tmp_filename = "tmp_#{zip_prefix}_#{Time.now.to_i}.zip"
+    origin_zip = Dir.glob("#{backup_folder}/#{zip_prefix}*").first
+
+    # add all files from original zip to a big hash
+    zip_files = {}
+    ZIPLIB.open(origin_zip) do |zf|
+      zf.entries.each { |zf_entry| zip_files[zf_entry.name] = zf.read(zf_entry.name) }
+    end
+
+    # modify the hash on every version
+    Dir.glob("#{backup_folder}/#{version_prefix}*").sort.first(versions).each do |version_file|
+      diff_hash = prepare_diff version_file
+
+      zip_files.each do |k, v|
+        next if diff_hash[k].nil?
+
+        if zip_files[k].nil?
+          diff_hash.delete k
+          next
+        end
+
+        diff_hash[k].find_all { |i| i[0] == "<" && i.include?("create_table") }.each do |deleted_table|
+          if zip_files["#{deleted_table.split("\"")[1]}.csv"].present?
+            zip_files["#{deleted_table.split("\"")[1]}.csv"] = nil # using delete will result in a stack level too deep
+          end
+        end if k == "schema.rb"
+
+        zip_files[k] = update_file v, diff_hash[k]
+
+        diff_hash.delete k
+      end
+
+      diff_hash.each { |k, v| zip_files[k] = update_file("", v) }
+    end
+
+    # save the big hash
+    ZIPLIB.open("#{backup_folder}/#{tmp_filename}", ZIPLIB::CREATE) do |f|
+      zip_files.each do |k, v|
+        f.get_output_stream(k) do |ff|
+          ff.write v
+          @tmp_files_compact << ff if ZIPOLD
+        end if v != "\n"
+      end
+    end
+
+    @tmp_files_compact.each { |i| File.delete i } if ZIPOLD
+    tmp_filename
+  end
+
+  # Returns a *hash* with of the parsed .diff file
+  def prepare_diff diff_path
+    diff_hash = {}
+
+    current_key = nil
+    File.read(diff_path).split("\n").each do |diff_line|
+      if diff_line.starts_with? "***************"
+        current_key = nil
+      elsif diff_line.starts_with? "*** "
+        current_key = diff_line.gsub("*", "").strip
+      elsif current_key.present?
+        diff_hash[current_key] ||= []
+        diff_hash[current_key] << diff_line
+      end
+    end
+
+    diff_hash
+  end
+
+  # It happens if there are two files with the same date(after using the date_format)
+  #
+  # Returns a *string* that will fix the duplicated filename issue by appending an index
+  def dup_file filename
+    return filename unless File.exist? filename
+    index = 1
+    name, ext = filename.split(".")
+
+    loop do
+      dup_filename = "#{name}_#{index}.#{ext}"
+      return dup_filename unless File.exist? dup_filename
+      index += 1
+    end
+  end
+
+  # Returns a *string* of the updated file after applying the diff
+  def update_file file, diff_lines
+    file = file.split "\n"
+    current_operation = nil
+    offset = 0
+
+    diff_lines.each_with_index do |diff_line, index|
+      if diff_line[0] != "<" && diff_line[0] != ">" && (diff_line.include?("a") || diff_line.include?("c") || diff_line.include?("d"))
+        # check linux diff ooutput to understand this
+        current_operation = diff_line
+
+        if current_operation.include? "a"
+          # Addition operation
+          l, r = diff_operation current_operation, "a"
+
+          insert_lines = []
+          loop do
+            break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?)
+            insert_lines << diff_lines[index][2..-1]  if diff_lines[index][0] == ">"
+            index += 1
+          end
+
+          file.insert offset + l.first + 1, *insert_lines
+          offset += [*insert_lines].length
+
+        elsif current_operation.include? "c"
+          # Changing operation
+          f, t = diff_operation current_operation, "c"
+
+          insert_lines = []
+          loop do
+            break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?)
+            insert_lines << diff_lines[index][2..-1]  if diff_lines[index][0] == ">"
+            index += 1
+          end
+
+          f.last.times { file.delete_at offset + f.first }
+          file.insert offset + f.first, *insert_lines
+          offset += [*insert_lines].length
+          offset -= f.last
+
+        elsif current_operation.include? "d"
+          # Deletion operation
+          r, l = diff_operation current_operation, "d"
+
+          r.last.times { file.delete_at offset + r.first }
+          offset -= r.last
+        end
+      end
+    end
+
+    file = file.map { |i| i.to_s.force_encoding("ASCII-8BIT") } # I HATE ENCODING
+    file.join("\n")  + "\n"
+  end
+
+  # Returns a *pair* *of* *integer* of the range interval in the diff operation
+  def diff_operation operation, sign
+    val1, val2 = operation.split sign
+    return diff_range(val1), diff_range(val2)
+  end
+
+
+  # Returns an *array* that contains the starting position of the change and how many operations it will take
+  def diff_range value
+    if value.include? ","
+      l, r = value.split(",")
+      [l.to_i - 1, r.to_i - l.to_i + 1]
+    else
+      [value.to_i - 1, 1]
+    end
+  end
+
+  # Two temporary files will be created, but deleted shortly after the operation is finished
+  #
+  # Returns an *array* that contains lines of the diff between +file1+ and +file2+
+  def diff_files file1, file2
+    diff_filename = "#{backup_folder}/diff"
+
+    File.open("#{diff_filename}1", "wb") { |f| f.write file1 }
+    File.open("#{diff_filename}2", "wb") { |f| f.write file2 }
+
+    diff_lines = []
+    IO.popen("diff #{diff_filename}1 #{diff_filename}2").each { |diff_line| diff_lines << diff_line }
+    # TODO: use something that doesn't depend on the operating system(that can work on Windows too)
+
+    File.delete("#{diff_filename}1")
+    File.delete("#{diff_filename}2")
+
+    diff_lines
+  end
+
+  # Returns a *string* of the current file created by backup_now function
+  def new_version_filename
+    backup_folder_files =  Dir.glob("#{backup_folder}/*").map { |i| i.gsub("#{backup_folder}/", "") }
+
+    zip_file = backup_folder_files.find { |i| i.starts_with?(zip_prefix) && i.ends_with?("zip") }
+    return "#{zip_prefix}_#{Time.now.strftime(date_format)}.zip" if zip_file.nil?
+
+    version_files = backup_folder_files.find_all { |i| i.starts_with?(version_prefix) && i.ends_with?("diff") }.sort
+    return "#{version_prefix}1_#{Time.now.strftime(date_format)}.diff" if version_files.blank?
+
+    last_version = version_files.last.gsub(version_prefix, "").split("_").first.to_i
+    return "#{version_prefix}#{last_version + 1}_#{Time.now.strftime(date_format)}.diff"
+  end
+
+  # Returns a *string* of parsed csv row to be inserted into database
+  def add_query values
+    values.map do |val|
+      if val.nil?
+        "NULL"
+      elsif val.is_a?(Date) || val.is_a?(DateTime) || val.is_a?(Time)
+        "\"#{val.strftime('%Y-%m-%d %H:%M:%S')}\"" # this should be a DB recognized format
+      elsif val.is_a?(Integer) || val.is_a?(Float) || val.is_a?(BigDecimal)
+        val
+      else
+        val.inspect
+      end
+    end.join(",") + "\n"
+  end
+
+  # Lock the library operations and stop the ActiveRecord logger
+  def lock
+    @logger = ActiveRecord::Base.logger
+    ActiveRecord::Base.logger = nil
+
+    if File.exist?("#{backup_folder}/.lock")
+      raise "Another operation is running. More info you can find in the '#{backup_folder}/.lock' file"
+    else
+      File.open("#{backup_folder}/.lock", "wb") do |f|
+        f.write caller.join("\n") + "\n"
+      end
+    end
+  end
+
+  # Unlock the library operations and restart ActiveRecord logger
+  def unlock
+    ActiveRecord::Base.logger = @logger
+
+    File.delete("#{backup_folder}/.lock") if File.exist?("#{backup_folder}/.lock")
+  end
+end
+include TinyBackup

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: tiny_backup
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Razvan Pavel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-02 00:00:00.000000000 Z
+dependencies: []
+description: Backup your database on S3 or local, optimized for small disk space.
+  The backup will have a starting zip with your data and diff files for each new backup
+email: pavelrazvan92@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/tiny_backup.rb
+homepage: http://rubygems.org/gems/tiny_backup
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Backup your database on S3 or local, optimized for small disk space
+test_files: []