travis-backup 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3a6957efa5907a32acc218a7534d65d93e6062850a05e739fc7a627b3262170
-  data.tar.gz: 2dbc87cb85ade0a9d5e474dc8febab32d42cd788e2dfbab8aa7d1431c623d2bf
+  metadata.gz: ce106f04b21145f1d238deb83a1aa1666f1a6861f0c5cbf09f58dd939a70b82c
+  data.tar.gz: f4b17655668028ee56c9fe3ecc88feb2baea37aa60c46435b1e9fe3ba251036c
 SHA512:
-  metadata.gz: 23b3148e39e799d9469c644eeec023fa07341e73d41ba68f29964977e36cd99a809545f5653433d456a770295a09f232aaa6a5bbe12e5b0914780d00da770ebd
-  data.tar.gz: 51ffab92a8164a5ccb0e2661a139705b8890b832df5370d9c32694a896c74c952234276ec6ec737b3cd3646e2b6541adece984fcd7c2d47d1b8b234f0d39c854
+  metadata.gz: c11bcb4f873d4dedc7f95eb706c9381f0f2e27f67762bd8ddbaa2d94d8689de1496a095e17e8cbcba42cfce04e4e9e61518bc63f3931495ceecae351f1674b12
+  data.tar.gz: 714f37c213a1c65d8e2d3b94653941c1c6e04919ce3c51750577fb4365ea976da0e90988deec914fed037df08cf6f640622928aaa5470ccb9bdbe69a35ca6045

data/.gitignore

@@ -5,3 +5,4 @@ dump/*
 !log/.keep
 !dump/.keep
 *.gem
+.byebug_history

data/README.md

@@ -1,6 +1,6 @@
 # README
 
-*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run in special modes to perform other specific tasks.
 
 ### Installation and run
 
@@ -29,6 +29,9 @@ All arguments:
   --move_logs                # run in move logs mode - move all logs to database at destination_db_url URL
   --destination_db_url URL   # URL for moving logs to
   --remove_orphans           # run in remove orphans mode
+  --orphans_table            # name of the table we will remove orphans from (if not defined, all tables are considered)
+  --load_from_files          # loads files stored in files_location to the database
+  --id_gap                   # concerns file loading - the gap between the biggest id in database and the lowest one that will be set to loaded data (that's for data inserted by other users during the load being performed; equals 1000 by default)
 ```
 
 Or inside your app:
@@ -60,9 +63,13 @@ backup.run(repo_id: 1)
 
 Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). When you run gem in this mode no files are created and no other tables are being touched.
 
-Using `--remove_orphans` flag you can remove all orphaned data from tables. When you run gem in this mode no files are created.
+Using `--remove_orphans` flag you can remove all orphaned data from the tables. You can pick a specific table using `--orphans_table` flag or, by leaving it undefined, let all tables to be processed in the removing orphans procedure. It can be combined with `--backup` flag in order to save removed data in files.
 
-Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with `--move_logs` or `--remove_orphans`.
+Using `--user_id`, `--org_id` or `--repo_id` flag without setting `--threshold` results in removing the specified user/organization/repository with all its dependencies. It can be combined with `--backup` flag in order to save removed data in files.
+
+Using `--load_from_files` flag you can restore dumped data from files located at path given by `--files_location`. The distance defined by `--id_gap` is going to be kept between biggest ids in the database and the lowest ones from the data loaded from files (and it equals 1000 by default).
+
+Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with special modes.
 
 ### Configuration options
 
@@ -80,9 +87,12 @@ backup:
   repo_id: 1                # run only for given repository
   move_logs: false          # run in move logs mode - move all logs to database at destination_db_url URL
   remove_orphans: false     # run in remove orphans mode
+  orphans_table: 'builds'   # name of the table we will remove orphans from (if not defined, all tables are considered)
+  load_from_files: false    # loads files stored in files_location to the database
+  id_gap: 1500              # concerns file loading - the gap between the biggest id in database and the lowest one that will be set to loaded data (that's for data inserted by other users during the load being performed; equals 1000 by default)
 ```
 
-You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`.
+You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`, `BACKUP_ORPHANS_TABLE`, `BACKUP_LOAD_FROM_FILES`, `BACKUP_ID_GAP`.
 
 You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema.
 

data/lib/backup/load_from_files.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require 'id_hash'
+
+class Backup
+  class LoadFromFiles
+    class JsonContent < String
+      def hash
+        @hash ||= JSON.parse(self).symbolize_keys
+      end
+    end
+
+    class DataFile
+      attr_accessor :content
+
+      def initialize(json_content)
+        @content = json_content
+      end
+
+      def table_name
+        @content.match(/"table_name":\s?"(\w+)"/)[1]
+      end
+
+      def table_name_sym
+        table_name.to_sym
+      end
+
+      def full_hash
+        @content.hash
+      end
+    end
+
+    class EntryFile < DataFile
+      def ids
+        @content.scan(/"id":\s?(\d+)/).flatten.map(&:to_i)
+      end
+
+      def lowest_id
+        ids.min
+      end
+
+      def entries
+        full_hash[:data]
+      end
+    end
+
+    class RelationshipFile < DataFile
+      def relationships
+        @relationships ||= full_hash[:nullified_relationships].map do |rel|
+          rel.symbolize_keys
+        end
+      end
+    end
+
+    def initialize(config, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+      @touched_models = []
+    end
+
+    def run
+      set_id_offsets
+      load_data_with_offsets
+      cancel_offset_for_foreign_data
+      set_id_sequences
+      load_nullified_relationships
+    end
+
+    private
+
+    def load_nullified_relationships
+      relationship_files.each do |file|
+        file.relationships.each do |rel|
+          offset = @id_offsets[file.table_name.to_sym]
+
+          ActiveRecord::Base.connection.execute(%{
+            update #{rel[:related_table]}
+            set #{rel[:foreign_key]} = #{rel[:parent_id].to_i + offset}
+            where id = #{rel[:related_id].to_i};
+          })
+        end
+      end
+    end
+
+    def set_id_sequences
+      @touched_models.each do |model|
+        value = model.last.id + 1
+        seq = model.table_name + '_id_seq'
+        set_sequence(seq, value)
+      end
+
+      set_shared_builds_tasks_seq
+    end
+
+    def set_shared_builds_tasks_seq
+      value = [Build.last&.id || -1, Job.last&.id || -1].max + 1
+
+      if value > 0
+        set_sequence("shared_builds_tasks_seq", value)
+      end
+    end
+
+    def set_sequence(seq, value)
+      ActiveRecord::Base.connection.execute("alter sequence #{seq} restart with #{value};")
+    end
+
+    def cancel_offset_for_foreign_data
+      @loaded_entries.each do |entry|
+        entry.class.reflect_on_all_associations.select { |a| a.macro == :belongs_to }.each do |association|
+          foreign_key = association.foreign_key.to_sym
+          if entry.send(association.name).nil? && entry.send(foreign_key).present?
+            entry_hash = entry.attributes.symbolize_keys
+            table_name = get_table_name(entry_hash, association)
+            offset = @id_offsets[table_name.to_sym]
+            next if offset.nil?
+
+            proper_id = entry.send(foreign_key) - offset
+            entry.update(foreign_key => proper_id)
+          end
+        end
+      end
+    end
+
+    def load_data_with_offsets
+      @repository_files = []
+
+      @loaded_entries = entry_files.map do |data_file|
+        model = Model.get_model_by_table_name(data_file.table_name)
+
+        if model == Repository
+          @repository_files << data_file
+          next
+        end
+
+        load_file(model, data_file)
+      end.flatten.compact
+
+      repository_entries = @repository_files.map do |data_file|
+        load_file(Repository, data_file)
+      end.flatten.compact
+
+      @loaded_entries.concat(repository_entries)
+    end
+
+    def load_file(model, data_file)
+      @touched_models << model
+
+      data_file.entries&.map do |entry_hash|
+        load_entry(model, entry_hash)
+      end
+    end
+
+    def load_entry(model, entry_hash)
+      entry_hash.symbolize_keys!
+      entry_hash[:id] += @id_offsets[model.table_name.to_sym]
+      add_offset_to_foreign_keys!(model, entry_hash)
+      model.create(entry_hash)
+    end
+
+    def add_offset_to_foreign_keys!(model, entry_hash)
+      model.reflect_on_all_associations.select { |a| a.macro == :belongs_to }.each do |association|
+        foreign_key_sym = association.foreign_key.to_sym
+        next unless entry_hash[foreign_key_sym]
+
+        table_name = get_table_name(entry_hash, association)
+        entry_hash[foreign_key_sym] += @id_offsets[table_name.to_sym] || 0
+      end
+    end
+
+    def get_table_name(entry_hash, association)
+      if association.polymorphic?
+        type_symbol = association.foreign_key.gsub(/_id$/, '_type').to_sym
+        class_name = entry_hash[type_symbol]
+      else
+        class_name = association.class_name
+      end
+
+      Model.get_model(class_name).table_name
+    end
+
+    def file_contents
+      @file_contents ||= Dir["#{@config.files_location}/**/*.json"].map do |file_path|
+        JsonContent.new(File.read(file_path))
+      end
+    end
+
+    def entry_files
+      @entry_files ||= file_contents.map do |content|
+        next unless content.hash[:data]
+
+        EntryFile.new(content)
+      end.compact
+    end
+
+    def relationship_files
+      @relationship_files ||= file_contents.map do |content|
+        next if content.hash[:data]
+
+        RelationshipFile.new(content)
+      end.compact
+    end
+
+    def find_lowest_ids_from_files
+      @lowest_ids_from_files = HashOfArrays.new
+
+      entry_files.each do |data_file|
+        table_name = data_file.table_name_sym
+        min_id = data_file.lowest_id
+        @lowest_ids_from_files.add(table_name, min_id) if min_id
+      end
+
+      @lowest_ids_from_files = @lowest_ids_from_files.map { |k, arr| [k, arr.min] }.to_h
+    end
+
+    def find_highest_ids_from_db
+      @highest_ids_from_db = {}
+
+      Model.subclasses.each do |model|
+        table_name = model.table_name.to_sym
+        @highest_ids_from_db[table_name] = model.order(:id).last&.id || 0
+      end
+    end
+
+    def set_id_offsets
+      find_lowest_ids_from_files
+      find_highest_ids_from_db
+
+      @id_offsets = @lowest_ids_from_files.map do |key, file_min|
+        db_max = @highest_ids_from_db[key]
+        offset = db_max - file_min + @config.id_gap
+        [key, offset]
+      end.to_h
+
+      make_offset_common_for_builds_and_jobs
+    end
+
+    def make_offset_common_for_builds_and_jobs
+      if @id_offsets[:builds] && @id_offsets[:jobs]
+        offset = [@id_offsets[:builds], @id_offsets[:jobs]].max
+        @id_offsets[:builds] = offset
+        @id_offsets[:jobs] = offset
+      end
+    end
+  end
+end

data/lib/backup/remove_orphans.rb

@@ -1,12 +1,20 @@
 # frozen_string_literal: true
 
+require 'byebug'
+require 'backup/save_id_hash_to_file'
+require 'backup/save_nullified_rels_to_file'
+
 class Backup
   class RemoveOrphans
+    include SaveIdHashToFile
+    include SaveNullifiedRelsToFile
+
     attr_reader :config
 
     def initialize(config, dry_run_reporter=nil)
       @config = config
       @dry_run_reporter = dry_run_reporter
+      @ids_to_remove = IdHash.new
     end
 
     def dry_run_report
@@ -14,19 +22,82 @@ class Backup
     end
 
     def run
+      if @config.orphans_table
+        check_specified(@config.orphans_table)
+      else
+        check_all
+      end
+
+      process_ids_to_remove
+    end
+
+    def check_all
       cases.each do |model_block|
-        model_block[:relations].each do |relation|
-          process_table(
-            main_model: model_block[:main_model],
-            related_model: relation[:related_model],
-            fk_name: relation[:fk_name],
-            method: model_block[:method],
-            dry_run_complement: model_block[:dry_run_complement]
-          )
-        end
+        check_model_block(model_block)
       end
     end
 
+    def check_specified(table_name)
+      model_block = cases.find { |c| c[:main_model] == Model.get_model_by_table_name(table_name) }
+      check_model_block(model_block)
+    end
+
+    def check_model_block(model_block)
+      model_block[:relations].each do |relation|
+        check_relationship(
+          main_model: model_block[:main_model],
+          related_model: relation[:related_model],
+          fk_name: relation[:fk_name],
+        )
+      end
+    end
+
+    def check_relationship(args)
+      main_model = args[:main_model]
+      related_model = args[:related_model]
+      fk_name = args[:fk_name]
+
+      main_table = main_model.table_name
+      related_table = related_model.table_name
+
+      for_delete = main_model.find_by_sql(%{
+        select a.*
+        from #{main_table} a
+        left join #{related_table} b
+        on a.#{fk_name} = b.id
+        where
+          a.#{fk_name} is not null
+          and b.id is null;
+      })
+
+      key = main_model.name.underscore.to_sym
+      ids = for_delete.map(&:id)
+      @ids_to_remove.add(key, *ids)
+    end
+
+    def process_ids_to_remove
+      return @dry_run_reporter.add_to_report(@ids_to_remove.with_table_symbols) if @config.dry_run
+
+      nullified_rels = nullify_builds_dependencies
+
+      if @config.if_backup
+        @subfolder = "remove_orphans_#{current_time_for_subfolder}"
+        save_nullified_rels_to_file(build: nullified_rels)
+        save_id_hash_to_file(@ids_to_remove)
+      end
+
+      @ids_to_remove.remove_entries_from_db
+    end
+
+    def nullify_builds_dependencies
+      nullified = @ids_to_remove[:build]&.map do |build_id|
+        build = Build.find(build_id)
+        build.nullify_all_dependencies
+      end
+
+      nullified&.flatten || []
+    end
+
     def cases
       [
         {
@@ -44,9 +115,7 @@ class Backup
             {related_model: PullRequest, fk_name: 'pull_request_id'},
             {related_model: Branch, fk_name: 'branch_id'},
             {related_model: Tag, fk_name: 'tag_id'}
-          ],
-          method: :destroy_all,
-          dry_run_complement: -> (ids) { add_builds_dependencies_to_dry_run_report(ids) }
+          ]
         }, {
           main_model: Job,
           relations: [
@@ -104,43 +173,5 @@ class Backup
         }
       ]
     end
-  
-    def add_builds_dependencies_to_dry_run_report(ids_for_delete)
-      repos_for_delete = Repository.where(current_build_id: ids_for_delete)
-      jobs_for_delete = Job.where(source_id: ids_for_delete)
-      @dry_run_reporter.add_to_report(:repositories, *repos_for_delete.map(&:id))
-      @dry_run_reporter.add_to_report(:jobs, *jobs_for_delete.map(&:id))
-    end
-  
-    def process_table(args)
-      main_model = args[:main_model]
-      related_model = args[:related_model]
-      fk_name = args[:fk_name]
-      method = args[:method] || :delete_all
-      dry_run_complement = args[:dry_run_complement]
-  
-      main_table = main_model.table_name
-      related_table = related_model.table_name
-  
-      for_delete = main_model.find_by_sql(%{
-        select a.*
-        from #{main_table} a
-        left join #{related_table} b
-        on a.#{fk_name} = b.id
-        where
-          a.#{fk_name} is not null
-          and b.id is null;
-      })
-  
-      ids_for_delete = for_delete.map(&:id)
-  
-      if config.dry_run
-        key = main_table.to_sym
-        @dry_run_reporter.add_to_report(key, *ids_for_delete)
-        dry_run_complement.call(ids_for_delete) if dry_run_complement
-      else
-        main_model.where(id: ids_for_delete).send(method)
-      end
-    end  
   end
 end

data/lib/backup/remove_specified/remove_heavy_data.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'backup/save_file'
+require 'backup/remove_specified/shared'
+
+class Backup
+  class RemoveSpecified
+    module RemoveHeavyData
+      include SaveIdHashToFile
+      include SaveNullifiedRelsToFile
+      include Shared
+
+      def remove_heavy_data_for_repos_owned_by(owner_id, owner_type)
+        Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
+          remove_heavy_data_for_repo(repository)
+        end
+      end
+
+      def remove_heavy_data_for_repo(repository)
+        remove_repo_builds(repository)
+        remove_repo_requests(repository)
+      end
+
+      def remove_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        threshold = @config.threshold.to_i.months.ago.to_datetime
+        builds_to_remove = repository.builds.where('created_at < ?', threshold)
+
+        builds_dependencies = builds_to_remove.map do |build|
+          result = build.ids_of_all_dependencies(dependencies_to_filter, :without_parents)
+          result.add(:build, build.id)
+          result
+        end.compact
+
+        ids_to_remove = IdHash.join(*builds_dependencies)
+        @subfolder = "repository_#{repository.id}_old_builds_#{current_time_for_subfolder}"
+
+        unless @config.dry_run
+          nullified_rels = builds_to_remove&.map(&:nullify_default_dependencies)&.flatten
+          save_nullified_rels_to_file(build: nullified_rels) if @config.if_backup
+        end
+
+        process_ids_to_remove(ids_to_remove)
+      end
+
+      def remove_repo_requests(repository)
+        threshold = @config.threshold.to_i.months.ago.to_datetime
+        requests_to_remove = repository.requests.where('created_at < ?', threshold)
+
+        requests_dependencies = requests_to_remove.map do |request|
+          hash_with_filtered = request.ids_of_all_dependencies(dependencies_to_filter, :without_parents)
+          hash_with_filtered.add(:request, request.id)
+        end
+
+        @subfolder = "repository_#{repository.id}_old_requests_#{current_time_for_subfolder}"
+
+        unless @config.dry_run
+          nullified_rels = requests_to_remove.map do |request|
+            nullify_filtered_dependencies(request)
+          end.flatten
+          save_nullified_rels_to_file(build: nullified_rels) if @config.if_backup
+        end
+
+        ids_to_remove = IdHash.join(*(requests_dependencies))
+        process_ids_to_remove(ids_to_remove)
+      end
+
+      private
+
+      def process_ids_to_remove(ids_to_remove)
+        if @config.dry_run
+          @dry_run_reporter.add_to_report(ids_to_remove.with_table_symbols)
+        else
+          save_id_hash_to_file(ids_to_remove) if @config.if_backup
+          ids_to_remove.remove_entries_from_db
+        end
+      end
+
+      def save_and_destroy_requests_batch(requests_batch, repository)
+        requests_export = requests_batch.map(&:attributes)
+        file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+        pretty_json = JSON.pretty_generate(requests_export)
+        if save_file(file_name, pretty_json)
+          destroy_requests_batch(requests_batch)
+        end
+        requests_export
+      end
+
+      def destroy_requests_batch(requests_batch)
+        return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+
+        requests_batch.each(&:destroy)
+      end
+
+      def destroy_requests_batch_dry(requests_batch)
+        @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id))
+      end
+    end
+  end
+end

data/lib/backup/remove_specified/remove_with_all_dependencies.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'backup/save_id_hash_to_file'
+require 'backup/save_nullified_rels_to_file'
+require 'models/user'
+require 'models/repository'
+require 'backup/remove_specified/shared'
+
+class Backup
+  class RemoveSpecified
+    module RemoveWithAllDependencies
+      include SaveIdHashToFile
+      include SaveNullifiedRelsToFile
+      include Shared
+
+      def remove_user_with_dependencies(user_id)
+        remove_entry_with_dependencies(:user, user_id)
+      end
+
+      def remove_org_with_dependencies(org_id)
+        remove_entry_with_dependencies(:organization, org_id)
+      end
+
+      def remove_repo_with_dependencies(repo_id)
+        remove_entry_with_dependencies(:repository, repo_id)
+      end
+
+      private
+
+      def remove_entry_with_dependencies(model_name, id)
+        @subfolder = "#{model_name}_#{id}_#{current_time_for_subfolder}"
+        entry = Model.get_model(model_name).find(id)
+        hash_with_filtered = entry.ids_of_all_dependencies_with_filtered(dependencies_to_filter, :without_parents)
+        ids_to_remove = hash_with_filtered[:main]
+        ids_to_remove.add(model_name, id)
+
+        return @dry_run_reporter.add_to_report(ids_to_remove) if @config.dry_run
+
+        nullified_rels = { build: nullify_filtered_dependencies(entry) || [] }
+
+        if @config.if_backup
+          save_nullified_rels_to_file(nullified_rels)
+          save_id_hash_to_file(ids_to_remove)
+        end
+
+        ids_to_remove.remove_entries_from_db(as_last: [:build])
+          # order important because of foreign key constraint between builds and repos
+      end
+    end
+  end
+end

data/lib/backup/remove_specified/shared.rb

@@ -0,0 +1,20 @@
+class Backup
+  class RemoveSpecified
+    module Shared
+      private
+
+      def nullify_filtered_dependencies(entry)
+        hash_with_filtered = entry.ids_of_all_dependencies_with_filtered(dependencies_to_filter)
+        filtered_builds = hash_with_filtered[:filtered_out]&.[](:build)&.map { |id| Build.find(id) }
+        filtered_builds&.map(&:nullify_default_dependencies)&.flatten
+      end
+
+      def dependencies_to_filter
+        {
+          build: Build.default_dependencies_symbols_to_nullify
+        }
+      end
+    end
+  end
+end
+