travis-backup 0.1.2 → 0.3.0

data/lib/models/request.rb

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 require 'models/repository'
 
 class Request < Model
+  belongs_to :owner, polymorphic: true
+  belongs_to :sender, polymorphic: true
   belongs_to :repository
+  belongs_to :branch
+  belongs_to :pull_request
+  belongs_to :tag
+  belongs_to :commit
+  has_many :abuses
+  has_many :messages, as: :subject
+  has_many :jobs, as: :source
+  has_many :builds
 
   self.table_name = 'requests'
 end

data/lib/models/ssl_key.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class SslKey < Model
+  belongs_to :repository
   self.table_name = 'ssl_keys'
 end

data/lib/models/stage.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class Stage < Model
+  belongs_to :build
+  has_many :jobs
+
   self.table_name = 'stages'
 end

data/lib/models/star.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Star < Model
+  belongs_to :repository
+  belongs_to :user
+  self.table_name = 'stars'
+end

data/lib/models/subscription.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Subscription < Model
+  belongs_to :owner, polymorphic: true
+  has_many :invoices
+  self.table_name = 'subscriptions'
+end

data/lib/models/tag.rb

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class Tag < Model
+  belongs_to :last_build, foreign_key: :last_build_id, class_name: 'Build'
+  belongs_to :repository
+  has_many :builds
+  has_many :commits
+  has_many :requests
+
   self.table_name = 'tags'
 end

data/lib/models/token.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Token < Model
+  belongs_to :user
+  self.table_name = 'tokens'
+end

data/lib/models/trial.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Trial < Model
+  belongs_to :owner, polymorphic: true
+  has_many :trial_allowances
+  self.table_name = 'trials'
+end

data/lib/models/trial_allowance.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class TrialAllowance < Model
+  belongs_to :creator, polymorphic: true
+  belongs_to :trial
+  self.table_name = 'trial_allowances'
+end

data/lib/models/user.rb

@@ -1,7 +1,39 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
+require 'models/abuse'
+require 'models/subscription'
+require 'models/owner_group'
+require 'models/trial'
+require 'models/trial_allowance'
+require 'models/broadcast'
+require 'models/star'
+require 'models/permission'
+require 'models/token'
+require 'models/email'
+require 'models/membership'
+require 'models/user_beta_feature'
 
 class User < Model
+  has_many :builds_for_that_this_user_is_owner, as: :owner, class_name: 'Build'
+  has_many :builds_for_that_this_user_is_sender, as: :sender, class_name: 'Build'
+  has_many :repositories, as: :owner
+  has_many :jobs, as: :owner
+  has_many :requests_for_that_this_user_is_owner, as: :owner, class_name: 'Request'
+  has_many :abuses, as: :owner
+  has_many :subscriptions, as: :owner
+  has_many :owner_groups, as: :owner
+  has_many :trials, as: :owner
+  has_many :trial_allowances, as: :creator
+  has_many :broadcasts, as: :recipient
+  has_many :requests_for_that_this_user_is_sender, as: :sender, class_name: 'Request'
+
+  has_many :stars
+  has_many :permissions
+  has_many :tokens
+  has_many :emails
+  has_many :memberships
+  has_many :user_beta_features
+
   self.table_name = 'users'
 end

data/lib/models/user_beta_feature.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class UserBetaFeature < Model
+  belongs_to :user
+  self.table_name = 'user_beta_features'
+end

data/lib/nullify_dependencies.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module NullifyDependencies
+  def nullify_dependencies(dependencies_symbols_to_nullify)
+    dependencies_symbols_to_nullify.map do |symbol|
+      dependencies = self.send(symbol) # e.g. build.tags_for_that_this_build_is_last
+
+      dependencies.map do |entry|
+        foreign_key = self.class.reflect_on_association(symbol).foreign_key.to_sym
+        entry.update(foreign_key => nil) # e.g. tag.update(last_build_id: nil)
+        {
+          related_table: entry.class.table_name,
+          foreign_key: foreign_key,
+          parent_id: self.id,
+          related_id: entry.id
+        }
+      end
+    end.flatten
+  end
+  
+  def nullify_all_dependencies
+    nullify_dependencies(symbols_of_all_direct_dependencies)
+  end
+
+  def nullify_default_dependencies
+    nullify_dependencies(default_dependencies_symbols_to_nullify)
+  end
+
+  def default_dependencies_symbols_to_nullify
+    self.class.default_dependencies_symbols_to_nullify
+  end
+
+  def self.default_dependencies_symbols_to_nullify
+    raise "self.default_dependencies_symbols_to_nullify not implemented in the #{self.class} class"
+  end
+
+  def default_dependencies_to_nullify
+    default_dependencies_symbols_to_nullify.map do |symbol|
+      self.send(symbol).to_a
+    end.flatten(1)
+  end
+end

data/lib/travis-backup.rb

@@ -3,298 +3,72 @@
 require 'active_support/core_ext/array'
 require 'active_support/time'
 require 'config'
+require 'db_helper'
+require 'dry_run_reporter'
+require 'models/abuse'
+require 'models/annotation'
 require 'models/repository'
-require 'models/log'
 require 'models/branch'
-require 'models/tag'
+require 'models/broadcast'
+require 'models/build'
 require 'models/commit'
 require 'models/cron'
+require 'models/email'
+require 'models/invoice'
+require 'models/job'
+require 'models/log'
+require 'models/membership'
+require 'models/message'
+require 'models/organization'
+require 'models/owner_group'
+require 'models/permission'
 require 'models/pull_request'
-require 'models/ssl_key'
+require 'models/queueable_job'
+require 'models/repository'
 require 'models/request'
+require 'models/ssl_key'
 require 'models/stage'
+require 'models/star'
+require 'models/subscription'
+require 'models/tag'
+require 'models/token'
+require 'models/trial_allowance'
+require 'models/trial'
+require 'models/user_beta_feature'
+require 'models/user'
+require 'backup/move_logs'
+require 'backup/remove_orphans'
+require 'backup/load_from_files'
+require 'backup/remove_specified'
 
 # main travis-backup class
 class Backup
   attr_accessor :config
-  attr_reader :dry_run_report
 
   def initialize(config_args={})
     @config = Config.new(config_args)
+    @db_helper = DbHelper.new(@config)
 
     if @config.dry_run
-      @dry_run_report = {builds: [], jobs: [], logs: [], requests: []}
+      @dry_run_reporter = DryRunReporter.new
     end
-
-    connect_db
   end
 
-  def connect_db(url=@config.database_url)
-    ActiveRecord::Base.establish_connection(url)
+  def dry_run_report
+    @dry_run_reporter.report
   end
 
   def run(args={})
-    user_id = args[:user_id] || @config.user_id
-    repo_id = args[:repo_id] || @config.repo_id
-    org_id = args[:org_id] || @config.org_id
-
-    if user_id
-      owner_id = user_id
-      owner_type = 'User'
-    elsif org_id
-      owner_id = org_id
-      owner_type = 'Organization'
-    end
-
     if @config.move_logs
-      move_logs
+      Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run
+    elsif @config.load_from_files
+      Backup::LoadFromFiles.new(@config, @dry_run_reporter).run
     elsif @config.remove_orphans
-      remove_orphans
-    elsif owner_id
-      process_repos_for_owner(owner_id, owner_type)
-    elsif repo_id
-      process_repo_with_id(repo_id)
-    else
-      process_all_repos
-    end
-
-    print_dry_run_report if @config.dry_run
-  end
-
-  def process_repos_for_owner(owner_id, owner_type)
-    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-      process_repo(repository)
-    end
-  end
-
-  def process_repo_with_id(repo_id)
-    process_repo(Repository.find(repo_id))
-  end
-
-  def process_all_repos
-    Repository.order(:id).each do |repository|
-      process_repo(repository)
-    end
-  end
-
-  def print_dry_run_report
-    if @dry_run_report.to_a.map(&:second).flatten.empty?
-      puts 'Dry run active. No data would be removed in normal run.'
-    else
-      puts 'Dry run active. The following data would be removed in normal run:'
-
-      @dry_run_report.to_a.map(&:first).each do |symbol|
-        print_dry_run_report_line(symbol)
-      end
-    end
-  end
-
-  def print_dry_run_report_line(symbol)
-    puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any?
-  end
-
-  def process_repo(repository)
-    process_repo_builds(repository)
-    process_repo_requests(repository)
-  end
-
-  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    current_build_id = repository.current_build_id || -1
-    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
-    end.compact
-  end
-
-  def process_repo_requests(repository)
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    repository.requests.where('created_at < ?', threshold)
-              .in_groups_of(@config.limit.to_i, false).map do |requests_batch|
-      @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
-    end.compact
-  end
-
-  def move_logs
-    return move_logs_dry if config.dry_run
-
-    connect_db(@config.database_url)
-    Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
-      log_hashes = logs_batch.as_json
-      connect_db(@config.destination_db_url)
-
-      log_hashes.each do |log_hash|
-        new_log = Log.new(log_hash)
-        new_log.save!
-      end
-
-      connect_db(@config.database_url)
-
-      logs_batch.each(&:destroy)
-    end
-  end
-
-  def move_logs_dry
-    dry_run_report[:logs].concat(Log.order(:id).map(&:id))
-  end
-
-  def remove_orphans
-    remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id')
-    remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id')
-    remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id')
-    remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id')
-    remove_orphans_for_table(Build, 'builds', 'requests', 'request_id')
-    remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id')
-    remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id')
-    remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id')
-    remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id')
-    remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id')
-    remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id')
-    remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id')
-    remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id')
-    remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id')
-    remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id')
-    remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id')
-    remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id')
-    remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id')
-    remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id')
-    remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id')
-    remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id')
-    remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id')
-    remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id')
-    remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id')
-    remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id')
-    remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id')
-  end
-
-  def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
-    for_delete = model_class.find_by_sql(%{
-      select a.*
-      from #{table_a_name} a
-      left join #{table_b_name} b
-      on a.#{fk_name} = b.id
-      where
-        a.#{fk_name} is not null
-        and b.id is null;
-    })
-
-    if config.dry_run
-      key = table_a_name.to_sym
-      dry_run_report[key] = [] if dry_run_report[key].nil?
-      dry_run_report[key].concat(for_delete.map(&:id))
-      dry_run_report[key].uniq!
+      Backup::RemoveOrphans.new(@config, @dry_run_reporter).run
     else
-      model_class.where(id: for_delete.map(&:id)).destroy_all
-    end
-  end
-
-  private
-
-  def save_and_destroy_builds_batch(builds_batch, repository)
-    builds_export = export_builds(builds_batch)
-    file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
-    pretty_json = JSON.pretty_generate(builds_export)
-    if save_file(file_name, pretty_json)
-      destroy_builds_batch(builds_batch)
+      Backup::RemoveSpecified.new(@config, @dry_run_reporter).run(args)
     end
-    builds_export
-  end
-
-  def destroy_builds_batch(builds_batch)
-    return destroy_builds_batch_dry(builds_batch) if @config.dry_run
-
-    builds_batch.each(&:destroy)
-  end
 
-  def destroy_builds_batch_dry(builds_batch)
-    @dry_run_report[:builds].concat(builds_batch.map(&:id))
-
-    jobs = builds_batch.map do |build|
-      build.jobs.map(&:id) || []
-    end.flatten
-
-    @dry_run_report[:jobs].concat(jobs)
-
-    logs = builds_batch.map do |build|
-      build.jobs.map do |job|
-        job.logs.map(&:id) || []
-      end.flatten || []
-    end.flatten
-
-    @dry_run_report[:logs].concat(logs)
-  end
-
-  def save_and_destroy_requests_batch(requests_batch, repository)
-    requests_export = export_requests(requests_batch)
-    file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
-    pretty_json = JSON.pretty_generate(requests_export)
-    if save_file(file_name, pretty_json)
-      destroy_requests_batch(requests_batch)
-    end
-    requests_export
-  end
-
-  def destroy_requests_batch(requests_batch)
-    return destroy_requests_batch_dry(requests_batch) if @config.dry_run
-
-    requests_batch.each(&:destroy)
-  end
-
-  def destroy_requests_batch_dry(requests_batch)
-    @dry_run_report[:requests].concat(requests_batch.map(&:id))
-  end
-
-  def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
-    return true if @config.dry_run
-
-    saved = false
-    begin
-      unless File.directory?(@config.files_location)
-        FileUtils.mkdir_p(@config.files_location)
-      end
-
-      File.open(file_path(file_name), 'w') do |file|
-        file.write(content)
-        file.close
-        saved = true
-      end
-    rescue => e
-      print "Failed to save #{file_name}, error: #{e.inspect}\n"
-    end
-    saved
-  end
-
-  def file_path(file_name)
-    "#{@config.files_location}/#{file_name}"
-  end
-
-  def export_builds(builds)
-    builds.map do |build|
-      build_export = build.attributes
-      build_export[:jobs] = export_jobs(build.jobs)
-
-      build_export
-    end
-  end
-
-  def export_jobs(jobs)
-    jobs.map do |job|
-      job_export = job.attributes
-      job_export[:logs] = export_logs(job.logs)
-
-      job_export
-    end
-  end
-
-  def export_logs(logs)
-    logs.map do |log|
-      log.attributes
-    end
-  end
-
-  def export_requests(requests)
-    requests.map do |request|
-      request.attributes
-    end
+    @dry_run_reporter.print_report if @config.dry_run
   end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.1.2'
+  s.version               = '0.3.0'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
@@ -26,4 +26,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'listen'
   s.add_development_dependency 'rubocop', '~> 0.75.1'
   s.add_development_dependency 'rubocop-rspec'
+  s.add_development_dependency 'database_cleaner-active_record'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-13 00:00:00.000000000 Z
+date: 2021-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -192,24 +192,37 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner-active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: 
 executables:
 - setup
-- rails
+- console
+- rake
+- travis_backup
 - spring
-- yarn
+- rails
 - bundle
-- travis_backup
-- rake
-- console
+- yarn
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - README.md
 - Rakefile
 - bin/bundle
@@ -247,22 +260,53 @@ files:
 - config/spring.rb
 - db/schema.sql
 - dump/.keep
+- lib/backup/load_from_files.rb
+- lib/backup/move_logs.rb
+- lib/backup/remove_orphans.rb
+- lib/backup/remove_specified.rb
+- lib/backup/remove_specified/remove_heavy_data.rb
+- lib/backup/remove_specified/remove_with_all_dependencies.rb
+- lib/backup/remove_specified/shared.rb
+- lib/backup/save_file.rb
+- lib/backup/save_id_hash_to_file.rb
+- lib/backup/save_nullified_rels_to_file.rb
 - lib/config.rb
+- lib/db_helper.rb
+- lib/dry_run_reporter.rb
+- lib/id_hash.rb
+- lib/ids_of_all_dependencies.rb
+- lib/model.rb
+- lib/models/abuse.rb
+- lib/models/annotation.rb
 - lib/models/branch.rb
+- lib/models/broadcast.rb
 - lib/models/build.rb
 - lib/models/commit.rb
 - lib/models/cron.rb
+- lib/models/email.rb
+- lib/models/invoice.rb
 - lib/models/job.rb
 - lib/models/log.rb
-- lib/models/model.rb
+- lib/models/membership.rb
+- lib/models/message.rb
 - lib/models/organization.rb
+- lib/models/owner_group.rb
+- lib/models/permission.rb
 - lib/models/pull_request.rb
+- lib/models/queueable_job.rb
 - lib/models/repository.rb
 - lib/models/request.rb
 - lib/models/ssl_key.rb
 - lib/models/stage.rb
+- lib/models/star.rb
+- lib/models/subscription.rb
 - lib/models/tag.rb
+- lib/models/token.rb
+- lib/models/trial.rb
+- lib/models/trial_allowance.rb
 - lib/models/user.rb
+- lib/models/user_beta_feature.rb
+- lib/nullify_dependencies.rb
 - lib/travis-backup.rb
 - log/.keep
 - package.json