travis-backup-for-v3 0.1.0

data/lib/backup/remove_specified/remove_heavy_data.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'backup/save_file'
+require 'backup/remove_specified/shared'
+
+class Backup
+  class RemoveSpecified
+    module RemoveHeavyData
+      include SaveIdHashToFile
+      include SaveNullifiedRelsToFile
+      include Shared
+
+      def remove_heavy_data_for_repos_owned_by(owner_id, owner_type)
+        Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
+          remove_heavy_data_for_repo(repository)
+        end
+      end
+
+      def remove_heavy_data_for_repo(repository)
+        remove_repo_builds(repository)
+        remove_repo_requests(repository)
+      end
+
+      def remove_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        threshold = @config.threshold.to_i.months.ago.to_datetime
+        builds_to_remove = repository.builds.where('created_at < ?', threshold)
+
+        builds_dependencies = builds_to_remove.map do |build|
+          result = build.ids_of_all_dependencies(dependencies_to_filter, :without_parents)
+          result.add(:build, build.id)
+          result
+        end.compact
+
+        ids_to_remove = IdHash.join(*builds_dependencies)
+        @subfolder = "repository_#{repository.id}_old_builds_#{current_time_for_subfolder}"
+
+        unless @config.dry_run
+          nullified_rels = builds_to_remove&.map(&:nullify_default_dependencies)&.flatten
+          save_nullified_rels_to_file(build: nullified_rels) if @config.if_backup
+        end
+
+        process_ids_to_remove(ids_to_remove)
+      end
+
+      def remove_repo_requests(repository)
+        threshold = @config.threshold.to_i.months.ago.to_datetime
+        requests_to_remove = repository.requests.where('created_at < ?', threshold)
+
+        requests_dependencies = requests_to_remove.map do |request|
+          hash_with_filtered = request.ids_of_all_dependencies(dependencies_to_filter, :without_parents)
+          hash_with_filtered.add(:request, request.id)
+        end
+
+        @subfolder = "repository_#{repository.id}_old_requests_#{current_time_for_subfolder}"
+
+        unless @config.dry_run
+          nullified_rels = requests_to_remove.map do |request|
+            nullify_filtered_dependencies(request)
+          end.flatten
+          save_nullified_rels_to_file(build: nullified_rels) if @config.if_backup
+        end
+
+        ids_to_remove = IdHash.join(*(requests_dependencies))
+        process_ids_to_remove(ids_to_remove)
+      end
+
+      private
+
+      def process_ids_to_remove(ids_to_remove)
+        if @config.dry_run
+          @dry_run_reporter.add_to_report(ids_to_remove.with_table_symbols)
+        else
+          save_id_hash_to_file(ids_to_remove) if @config.if_backup
+          ids_to_remove.remove_entries_from_db
+        end
+      end
+
+      def save_and_destroy_requests_batch(requests_batch, repository)
+        requests_export = requests_batch.map(&:attributes)
+        file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+        pretty_json = JSON.pretty_generate(requests_export)
+        if save_file(file_name, pretty_json)
+          destroy_requests_batch(requests_batch)
+        end
+        requests_export
+      end
+
+      def destroy_requests_batch(requests_batch)
+        return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+
+        requests_batch.each(&:destroy)
+      end
+
+      def destroy_requests_batch_dry(requests_batch)
+        @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id))
+      end
+    end
+  end
+end

data/lib/backup/remove_specified/remove_with_all_dependencies.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'backup/save_id_hash_to_file'
+require 'backup/save_nullified_rels_to_file'
+require 'models/user'
+require 'models/repository'
+require 'backup/remove_specified/shared'
+
+class Backup
+  class RemoveSpecified
+    module RemoveWithAllDependencies
+      include SaveIdHashToFile
+      include SaveNullifiedRelsToFile
+      include Shared
+
+      def remove_user_with_dependencies(user_id)
+        remove_entry_with_dependencies(:user, user_id)
+      end
+
+      def remove_org_with_dependencies(org_id)
+        remove_entry_with_dependencies(:organization, org_id)
+      end
+
+      def remove_repo_with_dependencies(repo_id)
+        remove_entry_with_dependencies(:repository, repo_id)
+      end
+
+      private
+
+      def remove_entry_with_dependencies(model_name, id)
+        @subfolder = "#{model_name}_#{id}_#{current_time_for_subfolder}"
+        entry = Model.get_model(model_name).find(id)
+        hash_with_filtered = entry.ids_of_all_dependencies_with_filtered(dependencies_to_filter, :without_parents)
+        ids_to_remove = hash_with_filtered[:main]
+        ids_to_remove.add(model_name, id)
+
+        return @dry_run_reporter.add_to_report(ids_to_remove) if @config.dry_run
+
+        nullified_rels = { build: nullify_filtered_dependencies(entry) || [] }
+
+        if @config.if_backup
+          save_nullified_rels_to_file(nullified_rels)
+          save_id_hash_to_file(ids_to_remove)
+        end
+
+        ids_to_remove.remove_entries_from_db(as_last: [:build])
+          # order important because of foreign key constraint between builds and repos
+      end
+    end
+  end
+end

data/lib/backup/remove_specified/shared.rb

@@ -0,0 +1,20 @@
+class Backup
+  class RemoveSpecified
+    module Shared
+      private
+
+      def nullify_filtered_dependencies(entry)
+        hash_with_filtered = entry.ids_of_all_dependencies_with_filtered(dependencies_to_filter)
+        filtered_builds = hash_with_filtered[:filtered_out]&.[](:build)&.map { |id| Build.find(id) }
+        filtered_builds&.map(&:nullify_default_dependencies)&.flatten
+      end
+
+      def dependencies_to_filter
+        {
+          build: Build.default_dependencies_symbols_to_nullify
+        }
+      end
+    end
+  end
+end
+  

data/lib/backup/remove_specified.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'backup/remove_specified/remove_with_all_dependencies'
+require 'backup/remove_specified/remove_heavy_data'
+
+class Backup
+  class RemoveSpecified
+    include RemoveHeavyData
+    include RemoveWithAllDependencies
+
+    attr_reader :config
+
+    def initialize(config, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+    end
+
+    def dry_run_report
+      @dry_run_reporter.report
+    end
+
+    def run(args={})
+      user_id = args[:user_id] || @config.user_id
+      repo_id = args[:repo_id] || @config.repo_id
+      org_id = args[:org_id] || @config.org_id
+
+      if user_id
+        process_user(user_id)
+      elsif org_id
+        process_organization(org_id)
+      elsif repo_id
+        process_repo_with_id(repo_id)
+      else
+        process_all_repos
+      end
+    end
+
+    def process_user(user_id)
+      if @config.threshold
+        remove_heavy_data_for_repos_owned_by(user_id, 'User')
+      else
+        remove_user_with_dependencies(user_id)
+      end
+    end
+
+    def process_organization(org_id)
+      if @config.threshold
+        remove_heavy_data_for_repos_owned_by(org_id, 'Organization')
+      else
+        remove_org_with_dependencies(org_id)
+      end
+    end
+
+    def process_repo_with_id(repo_id)
+      if @config.threshold
+        remove_heavy_data_for_repo(Repository.find(repo_id))
+      else
+        remove_repo_with_dependencies(repo_id)
+      end
+    end
+
+    def process_all_repos
+      Repository.order(:id).each do |repository|
+        remove_heavy_data_for_repo(repository)
+      end
+    end
+  end
+end

data/lib/backup/save_file.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SaveFile
+  def save_file(file_path, content) # rubocop:disable Metrics/MethodLength
+    return true if @config.dry_run
+
+    saved = false
+    begin
+      ensure_path(file_path)
+
+      File.open(full_file_path(file_path), 'w') do |file|
+        file.write(content)
+        file.close
+        saved = true
+      end
+    rescue => e
+      print "Failed to save #{file_path}, error: #{e.inspect}\n"
+    end
+    saved
+  end
+
+  def current_time_for_subfolder
+    Time.now.to_s.parameterize.underscore
+  end
+
+  def ensure_path(file_path)
+    path = folder_path(file_path)
+      
+    unless File.directory?(path)
+      FileUtils.mkdir_p(path)
+    end  
+  end
+
+  def full_file_path(file_path)
+    "#{@config.files_location}/#{file_path}"
+  end
+
+  def folder_path(file_path)
+    result = full_file_path(file_path).split('/')
+    result.pop
+    result.join('/')
+  end
+end

data/lib/backup/save_id_hash_to_file.rb

@@ -0,0 +1,33 @@
+require 'backup/save_file'
+
+module SaveIdHashToFile
+  include SaveFile
+
+  def save_id_hash_to_file(id_hash)
+    id_hash.each do |name, ids|
+      ids.sort.each_slice(@config.limit.to_i) do |ids_batch|
+        save_ids_batch_to_file(name, ids_batch)
+      end
+    end
+  end
+
+  def save_ids_batch_to_file(name, ids_batch)
+    model = Model.get_model(name)
+
+    export = {}
+    export[:table_name] = model.table_name
+    export[:data] = ids_batch.map do |id|
+      get_exported_object(model, id)
+    end
+
+    content = JSON.pretty_generate(export)
+    file_name = "#{name}_#{ids_batch.first}-#{ids_batch.last}.json"
+    file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
+    save_file(file_path, content)
+  end
+
+  def get_exported_object(model, id)
+    object = model.find(id)
+    object.attributes
+  end
+end

data/lib/backup/save_nullified_rels_to_file.rb

@@ -0,0 +1,29 @@
+require 'backup/save_file'
+
+module SaveNullifiedRelsToFile
+  include SaveFile
+
+  def save_nullified_rels_to_file(rels_hash)
+    @file_index = 1
+
+    rels_hash.each do |name, rels|
+      rels&.compact&.each_slice(@config.limit.to_i) do |rels_batch|
+        save_rels_batch_to_file(name, rels_batch)
+      end
+    end
+  end
+
+  def save_rels_batch_to_file(name, rels_batch)
+    model = Model.get_model(name)
+
+    export = {}
+    export[:table_name] = model.table_name
+    export[:nullified_relationships] = rels_batch
+
+    content = JSON.pretty_generate(export)
+    file_name = "nullified_relationships/build_#{@file_index}.json"
+    @file_index += 1
+    file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
+    save_file(file_path, content)
+  end
+end

data/lib/config.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+require 'optparse'
+
+class Config
+  attr_accessor :if_backup,
+    :dry_run,
+    :limit,
+    :threshold,
+    :files_location,
+    :database_url,
+    :user_id,
+    :repo_id,
+    :org_id,
+    :move_logs,
+    :remove_orphans,
+    :orphans_table,
+    :destination_db_url,
+    :load_from_files,
+    :id_gap
+
+  def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+    set_values(args)
+    check_values
+  end
+
+  def set_values(args)
+    config = yaml_load('config/settings.yml')
+    connection_details = yaml_load('config/database.yml')
+    argv_opts = argv_options
+    @if_backup = first_not_nil(
+      args[:if_backup],
+      argv_opts[:if_backup],
+      ENV['IF_BACKUP'],
+      config.dig('backup', 'if_backup'),
+      true
+    )
+    @dry_run = first_not_nil(
+      args[:dry_run],
+      argv_opts[:dry_run],
+      ENV['BACKUP_DRY_RUN'],
+      config.dig('backup', 'dry_run'),
+      false
+    )
+    @limit = first_not_nil(
+      args[:limit],
+      argv_opts[:limit],
+      ENV['BACKUP_LIMIT'],
+      config.dig('backup', 'limit'),
+      1000
+    )
+    @threshold = first_not_nil(
+      args[:threshold],
+      argv_opts[:threshold],
+      ENV['BACKUP_THRESHOLD'],
+      config.dig('backup', 'threshold')
+    )
+    @files_location = first_not_nil(
+      args[:files_location],
+      argv_opts[:files_location],
+      ENV['BACKUP_FILES_LOCATION'],
+      config.dig('backup', 'files_location'),
+      './dump'
+    )
+    @database_url = first_not_nil(
+      args[:database_url],
+      argv_opts[:database_url],
+      ENV['DATABASE_URL'],
+      connection_details.dig(ENV['RAILS_ENV'])
+    )
+    @user_id = first_not_nil(
+      args[:user_id],
+      argv_opts[:user_id],
+      ENV['BACKUP_USER_ID'],
+      config.dig('backup', 'user_id')
+    )
+    @repo_id = first_not_nil(
+      args[:repo_id],
+      argv_opts[:repo_id],
+      ENV['BACKUP_REPO_ID'],
+      config.dig('backup', 'repo_id')
+    )
+    @org_id = first_not_nil(
+      args[:org_id],
+      argv_opts[:org_id],
+      ENV['BACKUP_ORG_ID'],
+      config.dig('backup', 'org_id')
+    )
+    @move_logs = first_not_nil(
+      args[:move_logs],
+      argv_opts[:move_logs],
+      ENV['BACKUP_MOVE_LOGS'],
+      config.dig('backup', 'move_logs'),
+      false
+    )
+    @remove_orphans = first_not_nil(
+      args[:remove_orphans],
+      argv_opts[:remove_orphans],
+      ENV['BACKUP_REMOVE_ORPHANS'],
+      config.dig('backup', 'remove_orphans'),
+      false
+    )
+    @orphans_table = first_not_nil(
+      args[:orphans_table],
+      argv_opts[:orphans_table],
+      ENV['BACKUP_ORPHANS_TABLE'],
+      config.dig('backup', 'orphans_table'),
+      false
+    )
+    @destination_db_url = first_not_nil(
+      args[:destination_db_url],
+      argv_opts[:destination_db_url],
+      ENV['BACKUP_DESTINATION_DB_URL'],
+      connection_details.dig(ENV['RAILS_ENV'], 'destination')
+    )
+    @load_from_files = first_not_nil(
+      args[:load_from_files],
+      argv_opts[:load_from_files],
+      ENV['BACKUP_LOAD_FROM_FILES'],
+      config.dig('backup', 'load_from_files'),
+      false
+    )
+    @id_gap = first_not_nil(
+      args[:id_gap],
+      argv_opts[:id_gap],
+      ENV['BACKUP_ID_GAP'],
+      config.dig('backup', 'id_gap'),
+      1000
+    )
+  end
+
+  def check_values
+    if !@move_logs && !@remove_orphans && !@threshold && !@user_id && !@org_id && !@repo_id && !@load_from_files
+      message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
+        "Threshold defines number of months from now.")
+      abort message
+    end
+
+    if !@database_url
+      message = abort_message("Please provide proper database URL.")
+      abort message
+    end
+
+    if (@move_logs && !@destination_db_url)
+      abort "\nFor moving logs you need to specify your destination database. Example usage:\n" +
+      "\n  $ bin/travis_backup 'postgres://source_url' --move_logs --destination_db_url 'postgres://destination_url'\n" +
+      "\nor using in code:\n" +
+      "\n  Backup.new(database_url: 'postgres://source_url', destination_db_url: 'postgres://destination_url', move_logs: true)\n" +
+      "\nYou can also set it using environment variables or configuration files.\n"  
+    end
+  end
+
+  def abort_message(intro)
+    "\n#{intro}\n\nExample usage:\n"+
+    "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
+    "\nor using in code:\n" +
+    "\n  Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" +
+    "\nYou can also set it using environment variables or configuration files.\n"
+  end
+
+  def argv_options
+    argv_copy = ARGV.clone
+    options = {}
+    OptionParser.new do |opt|
+      opt.on('-b', '--backup') { |o| options[:if_backup] = o }
+      opt.on('-d', '--dry_run') { |o| options[:dry_run] = o }
+      opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i }
+      opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i }
+      opt.on('-f', '--files_location X') { |o| options[:files_location] = o }
+      opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i }
+      opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
+      opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
+      opt.on('--move_logs') { |o| options[:move_logs] = o }
+      opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
+      opt.on('--orphans_table X') { |o| options[:orphans_table] = o }
+      opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
+      opt.on('--load_from_files') { |o| options[:load_from_files] = o }
+      opt.on('--id_gap X') { |o| options[:id_gap] = o.to_i }
+    end.parse!
+
+    options[:database_url] = ARGV.shift if ARGV[0]
+    argv_copy.each do |arg|
+      ARGV.push(arg)
+    end
+    options
+  end
+
+  def first_not_nil(*arr)
+    arr.compact.first
+  end
+
+  def yaml_load(url)
+    begin
+      YAML.load(File.open(url))
+    rescue => e
+      {}
+    end
+  end
+end

data/lib/db_helper.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DbHelper
+  def initialize(config)
+    @config = config
+    connect_db
+  end
+
+  def connect_db(config_or_url=@config.database_url)
+    ActiveRecord::Base.establish_connection(config_or_url)
+  end
+
+  def do_in_other_db(config_or_url)
+    saved_config = ActiveRecord::Base.connection_db_config
+    connect_db(config_or_url)
+    result = yield
+    connect_db(saved_config)
+    result
+  end
+
+  def do_without_triggers
+    ActiveRecord::Base.connection.execute('set session_replication_role = replica;')
+    result = yield
+    ActiveRecord::Base.connection.execute('set session_replication_role = default;')
+    result
+  end
+end

data/lib/dry_run_reporter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class DryRunReporter
+  attr_reader :report
+
+  def initialize
+    @report = {}
+  end
+
+  def add_to_report(*args)
+    if args.first.is_a?(Hash)
+      add_to_report_as_hash(args.first)
+    else
+      add_to_report_as_key_and_values(*args)
+    end
+  end
+
+  def print_report
+    if @report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
+    else
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @report.to_a.map(&:first).each do |symbol|
+        print_report_line(symbol)
+      end
+    end
+  end
+
+  private
+
+  def add_to_report_as_key_and_values(key, *values)
+    report[key] = [] if report[key].nil?
+    report[key].concat(values)
+    report[key].uniq!
+  end
+
+  def add_to_report_as_hash(hash)
+    hash.each do |key, array|
+      add_to_report_as_key_and_values(key, *array)
+    end
+  end
+
+  def print_report_line(symbol)
+    puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any?
+  end
+end