travis-backup 0.1.2 → 0.3.0

data/lib/backup/remove_specified/shared.rb

@@ -0,0 +1,20 @@
+class Backup
+  class RemoveSpecified
+    module Shared
+      private
+
+      def nullify_filtered_dependencies(entry)
+        hash_with_filtered = entry.ids_of_all_dependencies_with_filtered(dependencies_to_filter)
+        filtered_builds = hash_with_filtered[:filtered_out]&.[](:build)&.map { |id| Build.find(id) }
+        filtered_builds&.map(&:nullify_default_dependencies)&.flatten
+      end
+
+      def dependencies_to_filter
+        {
+          build: Build.default_dependencies_symbols_to_nullify
+        }
+      end
+    end
+  end
+end
+  

data/lib/backup/remove_specified.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'backup/remove_specified/remove_with_all_dependencies'
+require 'backup/remove_specified/remove_heavy_data'
+
+class Backup
+  class RemoveSpecified
+    include RemoveHeavyData
+    include RemoveWithAllDependencies
+
+    attr_reader :config
+
+    def initialize(config, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+    end
+
+    def dry_run_report
+      @dry_run_reporter.report
+    end
+
+    def run(args={})
+      user_id = args[:user_id] || @config.user_id
+      repo_id = args[:repo_id] || @config.repo_id
+      org_id = args[:org_id] || @config.org_id
+
+      if user_id
+        process_user(user_id)
+      elsif org_id
+        process_organization(org_id)
+      elsif repo_id
+        process_repo_with_id(repo_id)
+      else
+        process_all_repos
+      end
+    end
+
+    def process_user(user_id)
+      if @config.threshold
+        remove_heavy_data_for_repos_owned_by(user_id, 'User')
+      else
+        remove_user_with_dependencies(user_id)
+      end
+    end
+
+    def process_organization(org_id)
+      if @config.threshold
+        remove_heavy_data_for_repos_owned_by(org_id, 'Organization')
+      else
+        remove_org_with_dependencies(org_id)
+      end
+    end
+
+    def process_repo_with_id(repo_id)
+      if @config.threshold
+        remove_heavy_data_for_repo(Repository.find(repo_id))
+      else
+        remove_repo_with_dependencies(repo_id)
+      end
+    end
+
+    def process_all_repos
+      Repository.order(:id).each do |repository|
+        remove_heavy_data_for_repo(repository)
+      end
+    end
+  end
+end

data/lib/backup/save_file.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SaveFile
+  def save_file(file_path, content) # rubocop:disable Metrics/MethodLength
+    return true if @config.dry_run
+
+    saved = false
+    begin
+      ensure_path(file_path)
+
+      File.open(full_file_path(file_path), 'w') do |file|
+        file.write(content)
+        file.close
+        saved = true
+      end
+    rescue => e
+      print "Failed to save #{file_path}, error: #{e.inspect}\n"
+    end
+    saved
+  end
+
+  def current_time_for_subfolder
+    Time.now.to_s.parameterize.underscore
+  end
+
+  def ensure_path(file_path)
+    path = folder_path(file_path)
+      
+    unless File.directory?(path)
+      FileUtils.mkdir_p(path)
+    end  
+  end
+
+  def full_file_path(file_path)
+    "#{@config.files_location}/#{file_path}"
+  end
+
+  def folder_path(file_path)
+    result = full_file_path(file_path).split('/')
+    result.pop
+    result.join('/')
+  end
+end

data/lib/backup/save_id_hash_to_file.rb

@@ -0,0 +1,33 @@
+require 'backup/save_file'
+
+module SaveIdHashToFile
+  include SaveFile
+
+  def save_id_hash_to_file(id_hash)
+    id_hash.each do |name, ids|
+      ids.sort.each_slice(@config.limit.to_i) do |ids_batch|
+        save_ids_batch_to_file(name, ids_batch)
+      end
+    end
+  end
+
+  def save_ids_batch_to_file(name, ids_batch)
+    model = Model.get_model(name)
+
+    export = {}
+    export[:table_name] = model.table_name
+    export[:data] = ids_batch.map do |id|
+      get_exported_object(model, id)
+    end
+
+    content = JSON.pretty_generate(export)
+    file_name = "#{name}_#{ids_batch.first}-#{ids_batch.last}.json"
+    file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
+    save_file(file_path, content)
+  end
+
+  def get_exported_object(model, id)
+    object = model.find(id)
+    object.attributes
+  end
+end

data/lib/backup/save_nullified_rels_to_file.rb

@@ -0,0 +1,29 @@
+require 'backup/save_file'
+
+module SaveNullifiedRelsToFile
+  include SaveFile
+
+  def save_nullified_rels_to_file(rels_hash)
+    @file_index = 1
+
+    rels_hash.each do |name, rels|
+      rels&.compact&.each_slice(@config.limit.to_i) do |rels_batch|
+        save_rels_batch_to_file(name, rels_batch)
+      end
+    end
+  end
+
+  def save_rels_batch_to_file(name, rels_batch)
+    model = Model.get_model(name)
+
+    export = {}
+    export[:table_name] = model.table_name
+    export[:nullified_relationships] = rels_batch
+
+    content = JSON.pretty_generate(export)
+    file_name = "nullified_relationships/build_#{@file_index}.json"
+    @file_index += 1
+    file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
+    save_file(file_path, content)
+  end
+end

data/lib/config.rb

@@ -2,7 +2,7 @@
 require 'optparse'
 
 class Config
-  attr_reader :if_backup,
+  attr_accessor :if_backup,
     :dry_run,
     :limit,
     :threshold,
@@ -13,7 +13,10 @@ class Config
     :org_id,
     :move_logs,
     :remove_orphans,
-    :destination_db_url
+    :orphans_table,
+    :destination_db_url,
+    :load_from_files,
+    :id_gap
 
   def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
     set_values(args)
@@ -96,18 +99,40 @@ class Config
       config.dig('backup', 'remove_orphans'),
       false
     )
+    @orphans_table = first_not_nil(
+      args[:orphans_table],
+      argv_opts[:orphans_table],
+      ENV['BACKUP_ORPHANS_TABLE'],
+      config.dig('backup', 'orphans_table'),
+      false
+    )
     @destination_db_url = first_not_nil(
       args[:destination_db_url],
       argv_opts[:destination_db_url],
       ENV['BACKUP_DESTINATION_DB_URL'],
       connection_details.dig(ENV['RAILS_ENV'], 'destination')
     )
+    @load_from_files = first_not_nil(
+      args[:load_from_files],
+      argv_opts[:load_from_files],
+      ENV['BACKUP_LOAD_FROM_FILES'],
+      config.dig('backup', 'load_from_files'),
+      false
+    )
+    @id_gap = first_not_nil(
+      args[:id_gap],
+      argv_opts[:id_gap],
+      ENV['BACKUP_ID_GAP'],
+      config.dig('backup', 'id_gap'),
+      1000
+    )
   end
 
   def check_values
-    if !@move_logs && !@remove_orphans && !@threshold
+    if !@move_logs && !@remove_orphans && !@threshold && !@user_id && !@org_id && !@repo_id && !@load_from_files
       message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
-        "Threshold defines number of months from now.")
+        "Threshold defines number of months from now. Alternatively you can define user_id, org_id or repo_id " +
+        "to remove whole user, organization or repository with all dependencies.")
       abort message
     end
 
@@ -126,10 +151,12 @@ class Config
   end
 
   def abort_message(intro)
-    "\n#{intro} Example usage:\n"+
-    "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
+    "\n#{intro}\n\nExample usage:\n"+
+    "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6" +
+    "\n  $ bin/travis_backup 'postgres://my_database_url' --user_id 1\n" +
     "\nor using in code:\n" +
-    "\n  Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" +
+    "\n  Backup.new(database_url: 'postgres://my_database_url', threshold: 6)" +
+    "\n  Backup.new(database_url: 'postgres://my_database_url', user_id: 1)\n" +
     "\nYou can also set it using environment variables or configuration files.\n"
   end
 
@@ -147,7 +174,10 @@ class Config
       opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
       opt.on('--move_logs') { |o| options[:move_logs] = o }
       opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
+      opt.on('--orphans_table X') { |o| options[:orphans_table] = o }
       opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
+      opt.on('--load_from_files') { |o| options[:load_from_files] = o }
+      opt.on('--id_gap X') { |o| options[:id_gap] = o.to_i }
     end.parse!
 
     options[:database_url] = ARGV.shift if ARGV[0]

data/lib/db_helper.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DbHelper
+  def initialize(config)
+    @config = config
+    connect_db
+  end
+
+  def connect_db(config_or_url=@config.database_url)
+    ActiveRecord::Base.establish_connection(config_or_url)
+  end
+
+  def do_in_other_db(config_or_url)
+    saved_config = ActiveRecord::Base.connection_db_config
+    connect_db(config_or_url)
+    result = yield
+    connect_db(saved_config)
+    result
+  end
+
+  def do_without_triggers
+    ActiveRecord::Base.connection.execute('set session_replication_role = replica;')
+    result = yield
+    ActiveRecord::Base.connection.execute('set session_replication_role = default;')
+    result
+  end
+end

data/lib/dry_run_reporter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class DryRunReporter
+  attr_reader :report
+
+  def initialize
+    @report = {}
+  end
+
+  def add_to_report(*args)
+    if args.first.is_a?(Hash)
+      add_to_report_as_hash(args.first)
+    else
+      add_to_report_as_key_and_values(*args)
+    end
+  end
+
+  def print_report
+    if @report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
+    else
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @report.to_a.map(&:first).each do |symbol|
+        print_report_line(symbol)
+      end
+    end
+  end
+
+  private
+
+  def add_to_report_as_key_and_values(key, *values)
+    report[key] = [] if report[key].nil?
+    report[key].concat(values)
+    report[key].uniq!
+  end
+
+  def add_to_report_as_hash(hash)
+    hash.each do |key, array|
+      add_to_report_as_key_and_values(key, *array)
+    end
+  end
+
+  def print_report_line(symbol)
+    puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any?
+  end
+end

data/lib/id_hash.rb

@@ -0,0 +1,97 @@
+require 'model'
+
+class HashOfArrays < Hash
+  def initialize(hash = {})
+    hash.each do |key, array|
+      self[key] = hash[key].clone
+    end
+  end
+
+  def clone
+    self.class.new(self)
+  end
+
+  def subtract(hash)
+    result = self.clone
+    hash.each do |key, array|
+      next if result[key].nil?
+
+      array.each do |el|
+        result[key].delete(el)
+      end
+
+      result.delete(key) if result[key].empty?
+    end
+    result
+  end
+
+  def add(key, *values)
+    self[key] = [] if self[key].nil?
+    self[key].concat(values)
+  end
+
+  def self.join(*hashes)
+    result = self.new
+    result.join(*hashes)
+  end
+
+  def join(*hashes)
+    hashes.each do |hash|
+      hash.each do |key, array|
+        self.add(key, *array)
+      end
+    end
+    self
+  end
+
+  def sort_arrays!
+    self.each do |key, array|
+      array.sort!
+    end
+  end
+end
+
+class IdHash < HashOfArrays
+  def add(key, *values)
+    super(key, *values)
+    self[key].uniq!
+    self
+  end
+
+  def remove_entries_from_db(as_first: [], as_last: [])
+    exceptionals = as_first + as_last
+    remove_from_exceptional(as_first)
+
+    self.each do |name, ids|
+      next if exceptionals.include?(name)
+      remove_entries_from_array(name, ids)
+    end
+
+    remove_from_exceptional(as_last)
+  end
+
+  def with_table_symbols
+    result = HashOfArrays.new
+
+    self.each do |name, ids|
+      symbol = Model.get_model(name).table_name.to_sym
+      result[symbol] = ids
+    end
+
+    result
+  end
+
+  private
+
+  def remove_from_exceptional(array)
+    array.each do |name|
+      ids = self[name]
+      remove_entries_from_array(name, ids)
+    end
+  end
+
+  def remove_entries_from_array(model_name, ids)
+    model = Model.get_model(model_name)
+    model.delete(ids) if model.present?
+  end
+end