travis-backup 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da5ede447a9a2a1b215d03d8582f62e8c4573e69e3337da91afaaffef07323f3
-  data.tar.gz: 01bc3c92f48f093ae3dd4ff23c07a0f2e9c5516b79c05279700d82570b0a2d40
+  metadata.gz: 85a9fa54e0b9f6637f1371e29d72682ff808bb2a1c7ee5e8d9432634f2961f14
+  data.tar.gz: 96f242b4475d5b50f5a46bfb0e2aaac618aad78a7107a89977d4e5ffde103eba
 SHA512:
-  metadata.gz: 775e8b95e35be972da5194d703d33645bfe21198d91f28efe2a417214d74e4fc71fee5a9c6d7a715ae2819b8d12a0b0bad691bca44b507782acd1dbf48c43d16
-  data.tar.gz: 289107dfc9d91b3cc43479c1a4973fe0fec4db91a519dd9cd8be321c9ce56f65e2170464ae3c88492385af5609d0ea52bfbe9d7510f1a44bcffa8f1dfea81b63
+  metadata.gz: 343866ae70df1f9f7c5fccdbc49f350c4d58ff195c9c527dd13995f3a66710b09ac53e269ab03eacaa36e6c0a17004d58635aec7c5ce0f7fce5d44c1029282de
+  data.tar.gz: f6f2a7d430da5bdfb6ece3de27c4c7201061071f6636e4d16f7dc3b70f04b922d1d0e80b87c78bdec33bdf42fa267c7d46ed733dc91eaccb8d094ec1e7005c22

data/README.md

@@ -1,6 +1,6 @@
 # README
 
-*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data.
 
 ### Installation and run
 
@@ -75,15 +75,19 @@ backup:
   limit: 1000               # builds limit for one backup file
   threshold: 6              # number of months from now - data younger than this time won't be backuped
   files_location: './dump'  # path of the folder in which backup files will be placed
-  user_id                   # run only for given user
-  org_id                    # run only for given organization
-  repo_id                   # run only for given repository
+  user_id: 1                # run only for given user
+  org_id: 1                 # run only for given organization
+  repo_id: 1                # run only for given repository
+  move_logs: false          # run in move logs mode - move all logs to database at destination_db_url URL
+  remove_orphans: false     # run in remove orphans mode
 ```
 
-You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`.
+You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`.
 
 You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema.
 
+For `move_logs` mode you need also to specify a destination database. You can set it also in `config/database.yml` file, in `destination` subsection, setting the `destination_db_url` hash argument while creating `Backup` instance or using the `BACKUP_DESTINATION_DB_URL` env var. Your destination database should be consistent with the Travis 3.0 database schema.
+
 ### How to run the test suite
 
 You can run the test after cloning this repository. Next you should call

data/lib/backup/move_logs.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class Backup
+  class MoveLogs
+    attr_reader :config
+
+    def initialize(config, db_helper, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+      @db_helper = db_helper
+    end
+
+    def run
+      return run_dry if @config.dry_run
+
+      @db_helper.connect_db(@config.database_url)
+      Log.order(:id).in_batches(of: @config.limit.to_i).map do |logs_batch|
+        log_hashes = logs_batch.as_json
+        @db_helper.connect_db(@config.destination_db_url)
+
+        log_hashes.each do |log_hash|
+          new_log = Log.new(log_hash)
+          new_log.save!
+        end
+
+        @db_helper.connect_db(@config.database_url)
+
+        logs_batch.each(&:destroy)
+      end
+    end
+
+    def run_dry
+      ids = Log.order(:id).map(&:id)
+      @dry_run_reporter.add_to_report(:logs, *ids)
+    end  
+  end
+end

data/lib/backup/remove_old.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+class Backup
+  class RemoveOld
+    attr_reader :config
+
+    def initialize(config, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+    end
+
+    def dry_run_report
+      @dry_run_reporter.report
+    end
+
+    def run(args={})
+      user_id = args[:user_id] || @config.user_id
+      repo_id = args[:repo_id] || @config.repo_id
+      org_id = args[:org_id] || @config.org_id
+
+      if user_id
+        process_repos_for_owner(user_id, 'User')
+      elsif org_id
+        process_repos_for_owner(org_id, 'Organization')
+      elsif repo_id
+        process_repo_with_id(repo_id)
+      else
+        process_all_repos
+      end  
+    end
+
+    def process_repos_for_owner(owner_id, owner_type)
+      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
+        process_repo(repository)
+      end
+    end
+  
+    def process_repo_with_id(repo_id)
+      process_repo(Repository.find(repo_id))
+    end
+  
+    def process_all_repos
+      Repository.order(:id).each do |repository|
+        process_repo(repository)
+      end
+    end
+  
+    def process_repo(repository)
+      process_repo_builds(repository)
+      process_repo_requests(repository)
+    end
+  
+    def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      threshold = @config.threshold.to_i.months.ago.to_datetime
+      current_build_id = repository.current_build_id || -1
+      repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
+                .in_batches(of: @config.limit.to_i).map do |builds_batch|
+        if @config.if_backup
+          file_prefix = "repository_#{repository.id}"
+          save_and_destroy_builds_batch(builds_batch, file_prefix)
+        else
+          destroy_builds_batch(builds_batch)
+        end
+      end.compact.reduce(&:&)
+    end
+  
+    def process_repo_requests(repository)
+      threshold = @config.threshold.to_i.months.ago.to_datetime
+      repository.requests.where('created_at < ?', threshold)
+                .in_batches(of: @config.limit.to_i).map do |requests_batch|
+        @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
+      end.compact
+    end
+  
+    private
+  
+    def save_and_destroy_builds_batch(builds_batch, file_prefix)
+      builds_export = builds_batch.map(&:attributes)
+  
+      dependencies_saved = builds_batch.map do |build|
+        save_build_jobs_and_logs(build, file_prefix)
+      end.reduce(&:&)
+  
+      if dependencies_saved
+        file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
+        pretty_json = JSON.pretty_generate(builds_export)
+        save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false
+      else
+        false
+      end
+    end
+  
+    def save_build_jobs_and_logs(build, file_prefix)
+      build.jobs.in_batches(of: @config.limit.to_i).map do |jobs_batch|
+        file_prefix = "#{file_prefix}_build_#{build.id}"
+        save_jobs_batch(jobs_batch, file_prefix)
+      end.compact.reduce(&:&)
+    end
+  
+    def save_jobs_batch(jobs_batch, file_prefix)
+      jobs_export = jobs_batch.map(&:attributes)
+  
+      logs_saved = jobs_batch.map do |job|
+        save_job_logs(job, file_prefix)
+      end.reduce(&:&)
+  
+      if logs_saved
+        file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json"
+        pretty_json = JSON.pretty_generate(jobs_export)
+        save_file(file_name, pretty_json)
+      else
+        false
+      end
+    end
+  
+    def save_job_logs(job, file_prefix)
+      job.logs.in_batches(of: @config.limit.to_i).map do |logs_batch|
+        file_prefix = "#{file_prefix}_job_#{job.id}"
+        save_logs_batch(logs_batch, file_prefix)
+      end.compact.reduce(&:&)
+    end
+  
+    def save_logs_batch(logs_batch, file_prefix)
+      logs_export = logs_batch.map(&:attributes)
+      file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json"
+      pretty_json = JSON.pretty_generate(logs_export)
+      save_file(file_name, pretty_json)
+    end
+  
+    def destroy_builds_batch(builds_batch)
+      return destroy_builds_batch_dry(builds_batch) if @config.dry_run
+  
+      builds_batch.each(&:destroy)
+    end
+  
+    def destroy_builds_batch_dry(builds_batch)
+      @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id))
+  
+      jobs_ids = builds_batch.map do |build|
+        build.jobs.map(&:id) || []
+      end.flatten
+  
+      @dry_run_reporter.add_to_report(:jobs, *jobs_ids)
+  
+      logs_ids = builds_batch.map do |build|
+        build.jobs.map do |job|
+          job.logs.map(&:id) || []
+        end.flatten || []
+      end.flatten
+  
+      @dry_run_reporter.add_to_report(:logs, *logs_ids)
+    end
+  
+    def save_and_destroy_requests_batch(requests_batch, repository)
+      requests_export = export_requests(requests_batch)
+      file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+      pretty_json = JSON.pretty_generate(requests_export)
+      if save_file(file_name, pretty_json)
+        destroy_requests_batch(requests_batch)
+      end
+      requests_export
+    end
+  
+    def destroy_requests_batch(requests_batch)
+      return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+  
+      requests_batch.each(&:destroy)
+    end
+  
+    def destroy_requests_batch_dry(requests_batch)
+      @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id))
+    end
+  
+    def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
+      return true if @config.dry_run
+  
+      saved = false
+      begin
+        unless File.directory?(@config.files_location)
+          FileUtils.mkdir_p(@config.files_location)
+        end
+  
+        File.open(file_path(file_name), 'w') do |file|
+          file.write(content)
+          file.close
+          saved = true
+        end
+      rescue => e
+        print "Failed to save #{file_name}, error: #{e.inspect}\n"
+      end
+      saved
+    end
+  
+    def file_path(file_name)
+      "#{@config.files_location}/#{file_name}"
+    end
+  
+    def export_requests(requests)
+      requests.map do |request|
+        request.attributes
+      end
+    end    
+  end
+end

data/lib/backup/remove_orphans.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+class Backup
+  class RemoveOrphans
+    attr_reader :config
+
+    def initialize(config, dry_run_reporter=nil)
+      @config = config
+      @dry_run_reporter = dry_run_reporter
+    end
+
+    def dry_run_report
+      @dry_run_reporter.report
+    end
+
+    def run
+      cases.each do |model_block|
+        model_block[:relations].each do |relation|
+          process_table(
+            main_model: model_block[:main_model],
+            related_model: relation[:related_model],
+            fk_name: relation[:fk_name],
+            method: model_block[:method],
+            dry_run_complement: model_block[:dry_run_complement]
+          )
+        end
+      end
+    end
+
+    def cases
+      [
+        {
+          main_model: Repository,
+          relations: [
+            {related_model: Build, fk_name: 'current_build_id'},
+            {related_model: Build, fk_name: 'last_build_id'}
+          ]
+        }, {
+          main_model: Build,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'},
+            {related_model: Commit, fk_name: 'commit_id'},
+            {related_model: Request, fk_name: 'request_id'},
+            {related_model: PullRequest, fk_name: 'pull_request_id'},
+            {related_model: Branch, fk_name: 'branch_id'},
+            {related_model: Tag, fk_name: 'tag_id'}
+          ],
+          method: :destroy_all,
+          dry_run_complement: -> (ids) { add_builds_dependencies_to_dry_run_report(ids) }
+        }, {
+          main_model: Job,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'},
+            {related_model: Commit, fk_name: 'commit_id'},
+            {related_model: Stage, fk_name: 'stage_id'},
+          ]
+        }, {
+          main_model: Branch,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'},
+            {related_model: Build, fk_name: 'last_build_id'}
+          ]
+        }, {
+          main_model: Tag,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'},
+            {related_model: Build, fk_name: 'last_build_id'}
+          ]
+        }, {
+          main_model: Commit,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'},
+            {related_model: Branch, fk_name: 'branch_id'},
+            {related_model: Tag, fk_name: 'tag_id'}
+          ]
+        }, {
+          main_model: Cron,
+          relations: [
+            {related_model: Branch, fk_name: 'branch_id'}
+          ]
+        }, {
+          main_model: PullRequest,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'}
+          ]
+        }, {
+          main_model: SslKey,
+          relations: [
+            {related_model: Repository, fk_name: 'repository_id'}
+          ]
+        }, {
+          main_model: Request,
+          relations: [
+            {related_model: Commit, fk_name: 'commit_id'},
+            {related_model: PullRequest, fk_name: 'pull_request_id'},
+            {related_model: Branch, fk_name: 'branch_id'},
+            {related_model: Tag, fk_name: 'tag_id'}
+          ]
+        }, {
+          main_model: Stage,
+          relations: [
+            {related_model: Build, fk_name: 'build_id'}
+          ]
+        }
+      ]
+    end
+  
+    def add_builds_dependencies_to_dry_run_report(ids_for_delete)
+      repos_for_delete = Repository.where(current_build_id: ids_for_delete)
+      jobs_for_delete = Job.where(source_id: ids_for_delete)
+      @dry_run_reporter.add_to_report(:repositories, *repos_for_delete.map(&:id))
+      @dry_run_reporter.add_to_report(:jobs, *jobs_for_delete.map(&:id))
+    end
+  
+    def process_table(args)
+      main_model = args[:main_model]
+      related_model = args[:related_model]
+      fk_name = args[:fk_name]
+      method = args[:method] || :delete_all
+      dry_run_complement = args[:dry_run_complement]
+  
+      main_table = main_model.table_name
+      related_table = related_model.table_name
+  
+      for_delete = main_model.find_by_sql(%{
+        select a.*
+        from #{main_table} a
+        left join #{related_table} b
+        on a.#{fk_name} = b.id
+        where
+          a.#{fk_name} is not null
+          and b.id is null;
+      })
+  
+      ids_for_delete = for_delete.map(&:id)
+  
+      if config.dry_run
+        key = main_table.to_sym
+        @dry_run_reporter.add_to_report(key, *ids_for_delete)
+        dry_run_complement.call(ids_for_delete) if dry_run_complement
+      else
+        main_model.where(id: ids_for_delete).send(method)
+      end
+    end  
+  end
+end

data/lib/config.rb

@@ -67,19 +67,19 @@ class Config
     @user_id = first_not_nil(
       args[:user_id],
       argv_opts[:user_id],
-      ENV['USER_ID'],
+      ENV['BACKUP_USER_ID'],
       config.dig('backup', 'user_id')
     )
     @repo_id = first_not_nil(
       args[:repo_id],
       argv_opts[:repo_id],
-      ENV['REPO_ID'],
+      ENV['BACKUP_REPO_ID'],
       config.dig('backup', 'repo_id')
     )
     @org_id = first_not_nil(
       args[:org_id],
       argv_opts[:org_id],
-      ENV['ORG_ID'],
+      ENV['BACKUP_ORG_ID'],
       config.dig('backup', 'org_id')
     )
     @move_logs = first_not_nil(

data/lib/db_helper.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DbHelper
+  def initialize(config)
+    @config = config
+    connect_db
+  end
+
+  def connect_db(config_or_url=@config.database_url)
+    ActiveRecord::Base.establish_connection(config_or_url)
+  end
+
+  def do_in_other_db(config_or_url)
+    saved_config = ActiveRecord::Base.connection_db_config
+    connect_db(config_or_url)
+    result = yield
+    connect_db(saved_config)
+    result
+  end
+
+  def do_without_triggers
+    ActiveRecord::Base.connection.execute('set session_replication_role = replica;')
+    result = yield
+    ActiveRecord::Base.connection.execute('set session_replication_role = default;')
+    result
+  end
+end

data/lib/dry_run_reporter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class DryRunReporter
+  attr_reader :report
+
+  def initialize
+    @report = {}
+  end
+
+  def add_to_report(key, *values)
+    report[key] = [] if report[key].nil?
+    report[key].concat(values)
+    report[key].uniq!
+  end
+
+  def print_report
+    if @report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
+    else
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @report.to_a.map(&:first).each do |symbol|
+        print_report_line(symbol)
+      end
+    end
+  end
+
+  private
+
+  def print_report_line(symbol)
+    puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any?
+  end
+end

data/lib/models/build.rb

@@ -8,6 +8,7 @@ require 'models/repository'
 class Build < Model
   belongs_to :repository
   has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job'
+  has_one    :repo_for_that_this_build_is_current, foreign_key: :current_build_id, dependent: :destroy, class_name: 'Repository'
 
   self.table_name = 'builds'
 end

data/lib/models/repository.rb

@@ -6,7 +6,7 @@ require 'models/request'
 
 # Repository model
 class Repository < Model
-  has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build'
+  has_many :builds, -> { order('id') }, foreign_key: :repository_id, class_name: 'Build'
   has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request'
 
   self.table_name = 'repositories'

data/lib/travis-backup.rb

@@ -3,6 +3,8 @@
 require 'active_support/core_ext/array'
 require 'active_support/time'
 require 'config'
+require 'db_helper'
+require 'dry_run_reporter'
 require 'models/repository'
 require 'models/log'
 require 'models/branch'
@@ -13,288 +15,36 @@ require 'models/pull_request'
 require 'models/ssl_key'
 require 'models/request'
 require 'models/stage'
+require 'backup/move_logs'
+require 'backup/remove_orphans'
+require 'backup/remove_old'
 
 # main travis-backup class
 class Backup
   attr_accessor :config
-  attr_reader :dry_run_report
 
   def initialize(config_args={})
     @config = Config.new(config_args)
+    @db_helper = DbHelper.new(@config)
 
     if @config.dry_run
-      @dry_run_report = {builds: [], jobs: [], logs: [], requests: []}
+      @dry_run_reporter = DryRunReporter.new
     end
-
-    connect_db
   end
 
-  def connect_db(url=@config.database_url)
-    ActiveRecord::Base.establish_connection(url)
+  def dry_run_report
+    @dry_run_reporter.report
   end
 
   def run(args={})
-    user_id = args[:user_id] || @config.user_id
-    repo_id = args[:repo_id] || @config.repo_id
-    org_id = args[:org_id] || @config.org_id
-
-    if user_id
-      owner_id = user_id
-      owner_type = 'User'
-    elsif org_id
-      owner_id = org_id
-      owner_type = 'Organization'
-    end
-
     if @config.move_logs
-      move_logs
+      Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run
     elsif @config.remove_orphans
-      remove_orphans
-    elsif owner_id
-      process_repos_for_owner(owner_id, owner_type)
-    elsif repo_id
-      process_repo_with_id(repo_id)
+      Backup::RemoveOrphans.new(@config, @dry_run_reporter).run
     else
-      process_all_repos
-    end
-
-    print_dry_run_report if @config.dry_run
-  end
-
-  def process_repos_for_owner(owner_id, owner_type)
-    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-      process_repo(repository)
-    end
-  end
-
-  def process_repo_with_id(repo_id)
-    process_repo(Repository.find(repo_id))
-  end
-
-  def process_all_repos
-    Repository.order(:id).each do |repository|
-      process_repo(repository)
-    end
-  end
-
-  def print_dry_run_report
-    if @dry_run_report.to_a.map(&:second).flatten.empty?
-      puts 'Dry run active. No data would be removed in normal run.'
-    else
-      puts 'Dry run active. The following data would be removed in normal run:'
-
-      @dry_run_report.to_a.map(&:first).each do |symbol|
-        print_dry_run_report_line(symbol)
-      end
-    end
-  end
-
-  def print_dry_run_report_line(symbol)
-    puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any?
-  end
-
-  def process_repo(repository)
-    process_repo_builds(repository)
-    process_repo_requests(repository)
-  end
-
-  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    current_build_id = repository.current_build_id || -1
-    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
-    end.compact
-  end
-
-  def process_repo_requests(repository)
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    repository.requests.where('created_at < ?', threshold)
-              .in_groups_of(@config.limit.to_i, false).map do |requests_batch|
-      @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
-    end.compact
-  end
-
-  def move_logs
-    return move_logs_dry if config.dry_run
-
-    connect_db(@config.database_url)
-    Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
-      log_hashes = logs_batch.as_json
-      connect_db(@config.destination_db_url)
-
-      log_hashes.each do |log_hash|
-        new_log = Log.new(log_hash)
-        new_log.save!
-      end
-
-      connect_db(@config.database_url)
-
-      logs_batch.each(&:destroy)
+      Backup::RemoveOld.new(@config, @dry_run_reporter).run(args)
     end
-  end
-
-  def move_logs_dry
-    dry_run_report[:logs].concat(Log.order(:id).map(&:id))
-  end
-
-  def remove_orphans
-    remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id')
-    remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id')
-    remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id')
-    remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id')
-    remove_orphans_for_table(Build, 'builds', 'requests', 'request_id')
-    remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id')
-    remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id')
-    remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id')
-    remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id')
-    remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id')
-    remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id')
-    remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id')
-    remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id')
-    remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id')
-    remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id')
-    remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id')
-    remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id')
-    remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id')
-    remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id')
-    remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id')
-    remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id')
-    remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id')
-    remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id')
-    remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id')
-    remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id')
-    remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id')
-  end
-
-  def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
-    for_delete = model_class.find_by_sql(%{
-      select a.*
-      from #{table_a_name} a
-      left join #{table_b_name} b
-      on a.#{fk_name} = b.id
-      where
-        a.#{fk_name} is not null
-        and b.id is null;
-    })
 
-    if config.dry_run
-      key = table_a_name.to_sym
-      dry_run_report[key] = [] if dry_run_report[key].nil?
-      dry_run_report[key].concat(for_delete.map(&:id))
-      dry_run_report[key].uniq!
-    else
-      model_class.where(id: for_delete.map(&:id)).delete_all
-    end
-  end
-
-  private
-
-  def save_and_destroy_builds_batch(builds_batch, repository)
-    builds_export = export_builds(builds_batch)
-    file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
-    pretty_json = JSON.pretty_generate(builds_export)
-    if save_file(file_name, pretty_json)
-      destroy_builds_batch(builds_batch)
-    end
-    builds_export
-  end
-
-  def destroy_builds_batch(builds_batch)
-    return destroy_builds_batch_dry(builds_batch) if @config.dry_run
-
-    builds_batch.each(&:destroy)
-  end
-
-  def destroy_builds_batch_dry(builds_batch)
-    @dry_run_report[:builds].concat(builds_batch.map(&:id))
-
-    jobs = builds_batch.map do |build|
-      build.jobs.map(&:id) || []
-    end.flatten
-
-    @dry_run_report[:jobs].concat(jobs)
-
-    logs = builds_batch.map do |build|
-      build.jobs.map do |job|
-        job.logs.map(&:id) || []
-      end.flatten || []
-    end.flatten
-
-    @dry_run_report[:logs].concat(logs)
-  end
-
-  def save_and_destroy_requests_batch(requests_batch, repository)
-    requests_export = export_requests(requests_batch)
-    file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
-    pretty_json = JSON.pretty_generate(requests_export)
-    if save_file(file_name, pretty_json)
-      destroy_requests_batch(requests_batch)
-    end
-    requests_export
-  end
-
-  def destroy_requests_batch(requests_batch)
-    return destroy_requests_batch_dry(requests_batch) if @config.dry_run
-
-    requests_batch.each(&:destroy)
-  end
-
-  def destroy_requests_batch_dry(requests_batch)
-    @dry_run_report[:requests].concat(requests_batch.map(&:id))
-  end
-
-  def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
-    return true if @config.dry_run
-
-    saved = false
-    begin
-      unless File.directory?(@config.files_location)
-        FileUtils.mkdir_p(@config.files_location)
-      end
-
-      File.open(file_path(file_name), 'w') do |file|
-        file.write(content)
-        file.close
-        saved = true
-      end
-    rescue => e
-      print "Failed to save #{file_name}, error: #{e.inspect}\n"
-    end
-    saved
-  end
-
-  def file_path(file_name)
-    "#{@config.files_location}/#{file_name}"
-  end
-
-  def export_builds(builds)
-    builds.map do |build|
-      build_export = build.attributes
-      build_export[:jobs] = export_jobs(build.jobs)
-
-      build_export
-    end
-  end
-
-  def export_jobs(jobs)
-    jobs.map do |job|
-      job_export = job.attributes
-      job_export[:logs] = export_logs(job.logs)
-
-      job_export
-    end
-  end
-
-  def export_logs(logs)
-    logs.map do |log|
-      log.attributes
-    end
-  end
-
-  def export_requests(requests)
-    requests.map do |request|
-      request.attributes
-    end
+    @dry_run_reporter.print_report if @config.dry_run
   end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.1.0'
+  s.version               = '0.2.0'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'listen'
   s.add_development_dependency 'rubocop', '~> 0.75.1'
   s.add_development_dependency 'rubocop-rspec'
+  s.add_development_dependency 'database_cleaner-active_record'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-07 00:00:00.000000000 Z
+date: 2021-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -192,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner-active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: 
 executables:
@@ -209,7 +223,6 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - README.md
 - Rakefile
 - bin/bundle
@@ -247,7 +260,12 @@ files:
 - config/spring.rb
 - db/schema.sql
 - dump/.keep
+- lib/backup/move_logs.rb
+- lib/backup/remove_old.rb
+- lib/backup/remove_orphans.rb
 - lib/config.rb
+- lib/db_helper.rb
+- lib/dry_run_reporter.rb
 - lib/models/branch.rb
 - lib/models/build.rb
 - lib/models/commit.rb

data/Gemfile.lock

@@ -1,212 +0,0 @@
-PATH
-  remote: .
-  specs:
-    travis-backup (0.1.0)
-      activerecord
-      bootsnap
-      pg
-      pry
-      rails
-      tzinfo-data
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    actioncable (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      nio4r (~> 2.0)
-      websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activejob (= 6.1.4.1)
-      activerecord (= 6.1.4.1)
-      activestorage (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      mail (>= 2.7.1)
-    actionmailer (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      actionview (= 6.1.4.1)
-      activejob (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      mail (~> 2.5, >= 2.5.4)
-      rails-dom-testing (~> 2.0)
-    actionpack (6.1.4.1)
-      actionview (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      rack (~> 2.0, >= 2.0.9)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activerecord (= 6.1.4.1)
-      activestorage (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      nokogiri (>= 1.8.5)
-    actionview (6.1.4.1)
-      activesupport (= 6.1.4.1)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.1.4.1)
-      activesupport (= 6.1.4.1)
-      globalid (>= 0.3.6)
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activestorage (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activejob (= 6.1.4.1)
-      activerecord (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      marcel (~> 1.0.0)
-      mini_mime (>= 1.1.0)
-    activesupport (6.1.4.1)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    ast (2.4.2)
-    bootsnap (1.7.7)
-      msgpack (~> 1.0)
-    brakeman (5.1.1)
-    builder (3.2.4)
-    byebug (11.1.3)
-    coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    crass (1.0.6)
-    diff-lcs (1.4.4)
-    erubi (1.10.0)
-    factory_bot (6.2.0)
-      activesupport (>= 5.0.0)
-    ffi (1.15.3)
-    globalid (0.5.2)
-      activesupport (>= 5.0)
-    i18n (1.8.10)
-      concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
-    listen (3.7.0)
-      rb-fsevent (~> 0.10, >= 0.10.3)
-      rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.12.0)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    mail (2.7.1)
-      mini_mime (>= 0.1.1)
-    marcel (1.0.1)
-    method_source (1.0.0)
-    mini_mime (1.1.1)
-    mini_portile2 (2.6.1)
-    minitest (5.14.4)
-    msgpack (1.4.2)
-    nio4r (2.5.8)
-    nokogiri (1.12.3)
-      mini_portile2 (~> 2.6.1)
-      racc (~> 1.4)
-    parallel (1.20.1)
-    parser (3.0.2.0)
-      ast (~> 2.4.1)
-    pg (1.2.3)
-    pry (0.14.1)
-      coderay (~> 1.1)
-      method_source (~> 1.0)
-    racc (1.5.2)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
-    rails (6.1.4.1)
-      actioncable (= 6.1.4.1)
-      actionmailbox (= 6.1.4.1)
-      actionmailer (= 6.1.4.1)
-      actionpack (= 6.1.4.1)
-      actiontext (= 6.1.4.1)
-      actionview (= 6.1.4.1)
-      activejob (= 6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activerecord (= 6.1.4.1)
-      activestorage (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      bundler (>= 1.15.0)
-      railties (= 6.1.4.1)
-      sprockets-rails (>= 2.0.0)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.1)
-      loofah (~> 2.3)
-    railties (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      method_source
-      rake (>= 0.13)
-      thor (~> 1.0)
-    rainbow (3.0.0)
-    rake (13.0.6)
-    rb-fsevent (0.11.0)
-    rb-inotify (0.10.1)
-      ffi (~> 1.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-rails (5.0.2)
-      actionpack (>= 5.2)
-      activesupport (>= 5.2)
-      railties (>= 5.2)
-      rspec-core (~> 3.10)
-      rspec-expectations (~> 3.10)
-      rspec-mocks (~> 3.10)
-      rspec-support (~> 3.10)
-    rspec-support (3.10.2)
-    rubocop (0.75.1)
-      jaro_winkler (~> 1.5.1)
-      parallel (~> 1.10)
-      parser (>= 2.6)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-rspec (1.41.0)
-      rubocop (>= 0.68.1)
-    ruby-progressbar (1.11.0)
-    sprockets (4.0.2)
-      concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
-    sprockets-rails (3.2.2)
-      actionpack (>= 4.0)
-      activesupport (>= 4.0)
-      sprockets (>= 3.0.0)
-    thor (1.1.0)
-    tzinfo (2.0.4)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.1)
-      tzinfo (>= 1.0.0)
-    unicode-display_width (1.6.1)
-    websocket-driver (0.7.5)
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-    zeitwerk (2.4.2)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  brakeman
-  byebug
-  factory_bot
-  listen
-  rspec-rails
-  rubocop (~> 0.75.1)
-  rubocop-rspec
-  travis-backup!
-
-BUNDLED WITH
-   2.1.4