travis-backup 0.2.1 → 0.3.0

data/lib/models/request.rb

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 require 'models/repository'
 
 class Request < Model
+  belongs_to :owner, polymorphic: true
+  belongs_to :sender, polymorphic: true
   belongs_to :repository
+  belongs_to :branch
+  belongs_to :pull_request
+  belongs_to :tag
+  belongs_to :commit
+  has_many :abuses
+  has_many :messages, as: :subject
+  has_many :jobs, as: :source
+  has_many :builds
 
   self.table_name = 'requests'
 end

data/lib/models/ssl_key.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class SslKey < Model
+  belongs_to :repository
   self.table_name = 'ssl_keys'
 end

data/lib/models/stage.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class Stage < Model
+  belongs_to :build
+  has_many :jobs
+
   self.table_name = 'stages'
 end

data/lib/models/star.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Star < Model
+  belongs_to :repository
+  belongs_to :user
+  self.table_name = 'stars'
+end

data/lib/models/subscription.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Subscription < Model
+  belongs_to :owner, polymorphic: true
+  has_many :invoices
+  self.table_name = 'subscriptions'
+end

data/lib/models/tag.rb

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
 
 class Tag < Model
+  belongs_to :last_build, foreign_key: :last_build_id, class_name: 'Build'
+  belongs_to :repository
+  has_many :builds
+  has_many :commits
+  has_many :requests
+
   self.table_name = 'tags'
 end

data/lib/models/token.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Token < Model
+  belongs_to :user
+  self.table_name = 'tokens'
+end

data/lib/models/trial.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class Trial < Model
+  belongs_to :owner, polymorphic: true
+  has_many :trial_allowances
+  self.table_name = 'trials'
+end

data/lib/models/trial_allowance.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class TrialAllowance < Model
+  belongs_to :creator, polymorphic: true
+  belongs_to :trial
+  self.table_name = 'trial_allowances'
+end

data/lib/models/user.rb

@@ -1,7 +1,39 @@
 # frozen_string_literal: true
 
-require 'models/model'
+require 'model'
+require 'models/abuse'
+require 'models/subscription'
+require 'models/owner_group'
+require 'models/trial'
+require 'models/trial_allowance'
+require 'models/broadcast'
+require 'models/star'
+require 'models/permission'
+require 'models/token'
+require 'models/email'
+require 'models/membership'
+require 'models/user_beta_feature'
 
 class User < Model
+  has_many :builds_for_that_this_user_is_owner, as: :owner, class_name: 'Build'
+  has_many :builds_for_that_this_user_is_sender, as: :sender, class_name: 'Build'
+  has_many :repositories, as: :owner
+  has_many :jobs, as: :owner
+  has_many :requests_for_that_this_user_is_owner, as: :owner, class_name: 'Request'
+  has_many :abuses, as: :owner
+  has_many :subscriptions, as: :owner
+  has_many :owner_groups, as: :owner
+  has_many :trials, as: :owner
+  has_many :trial_allowances, as: :creator
+  has_many :broadcasts, as: :recipient
+  has_many :requests_for_that_this_user_is_sender, as: :sender, class_name: 'Request'
+
+  has_many :stars
+  has_many :permissions
+  has_many :tokens
+  has_many :emails
+  has_many :memberships
+  has_many :user_beta_features
+
   self.table_name = 'users'
 end

data/lib/models/user_beta_feature.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'model'
+
+class UserBetaFeature < Model
+  belongs_to :user
+  self.table_name = 'user_beta_features'
+end

data/lib/nullify_dependencies.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module NullifyDependencies
+  def nullify_dependencies(dependencies_symbols_to_nullify)
+    dependencies_symbols_to_nullify.map do |symbol|
+      dependencies = self.send(symbol) # e.g. build.tags_for_that_this_build_is_last
+
+      dependencies.map do |entry|
+        foreign_key = self.class.reflect_on_association(symbol).foreign_key.to_sym
+        entry.update(foreign_key => nil) # e.g. tag.update(last_build_id: nil)
+        {
+          related_table: entry.class.table_name,
+          foreign_key: foreign_key,
+          parent_id: self.id,
+          related_id: entry.id
+        }
+      end
+    end.flatten
+  end
+  
+  def nullify_all_dependencies
+    nullify_dependencies(symbols_of_all_direct_dependencies)
+  end
+
+  def nullify_default_dependencies
+    nullify_dependencies(default_dependencies_symbols_to_nullify)
+  end
+
+  def default_dependencies_symbols_to_nullify
+    self.class.default_dependencies_symbols_to_nullify
+  end
+
+  def self.default_dependencies_symbols_to_nullify
+    raise "self.default_dependencies_symbols_to_nullify not implemented in the #{self.class} class"
+  end
+
+  def default_dependencies_to_nullify
+    default_dependencies_symbols_to_nullify.map do |symbol|
+      self.send(symbol).to_a
+    end.flatten(1)
+  end
+end

data/lib/travis-backup.rb

@@ -5,19 +5,41 @@ require 'active_support/time'
 require 'config'
 require 'db_helper'
 require 'dry_run_reporter'
+require 'models/abuse'
+require 'models/annotation'
 require 'models/repository'
-require 'models/log'
 require 'models/branch'
-require 'models/tag'
+require 'models/broadcast'
+require 'models/build'
 require 'models/commit'
 require 'models/cron'
+require 'models/email'
+require 'models/invoice'
+require 'models/job'
+require 'models/log'
+require 'models/membership'
+require 'models/message'
+require 'models/organization'
+require 'models/owner_group'
+require 'models/permission'
 require 'models/pull_request'
-require 'models/ssl_key'
+require 'models/queueable_job'
+require 'models/repository'
 require 'models/request'
+require 'models/ssl_key'
 require 'models/stage'
+require 'models/star'
+require 'models/subscription'
+require 'models/tag'
+require 'models/token'
+require 'models/trial_allowance'
+require 'models/trial'
+require 'models/user_beta_feature'
+require 'models/user'
 require 'backup/move_logs'
 require 'backup/remove_orphans'
-require 'backup/remove_old'
+require 'backup/load_from_files'
+require 'backup/remove_specified'
 
 # main travis-backup class
 class Backup
@@ -39,10 +61,12 @@ class Backup
   def run(args={})
     if @config.move_logs
       Backup::MoveLogs.new(@config, @db_helper, @dry_run_reporter).run
+    elsif @config.load_from_files
+      Backup::LoadFromFiles.new(@config, @dry_run_reporter).run
     elsif @config.remove_orphans
       Backup::RemoveOrphans.new(@config, @dry_run_reporter).run
     else
-      Backup::RemoveOld.new(@config, @dry_run_reporter).run(args)
+      Backup::RemoveSpecified.new(@config, @dry_run_reporter).run(args)
     end
 
     @dry_run_reporter.print_report if @config.dry_run

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.2.1'
+  s.version               = '0.3.0'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-04 00:00:00.000000000 Z
+date: 2021-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -210,13 +210,13 @@ description:
 email: 
 executables:
 - setup
-- rails
+- console
+- rake
+- travis_backup
 - spring
-- yarn
+- rails
 - bundle
-- travis_backup
-- rake
-- console
+- yarn
 extensions: []
 extra_rdoc_files: []
 files:
@@ -260,27 +260,53 @@ files:
 - config/spring.rb
 - db/schema.sql
 - dump/.keep
+- lib/backup/load_from_files.rb
 - lib/backup/move_logs.rb
-- lib/backup/remove_old.rb
 - lib/backup/remove_orphans.rb
+- lib/backup/remove_specified.rb
+- lib/backup/remove_specified/remove_heavy_data.rb
+- lib/backup/remove_specified/remove_with_all_dependencies.rb
+- lib/backup/remove_specified/shared.rb
+- lib/backup/save_file.rb
+- lib/backup/save_id_hash_to_file.rb
+- lib/backup/save_nullified_rels_to_file.rb
 - lib/config.rb
 - lib/db_helper.rb
 - lib/dry_run_reporter.rb
+- lib/id_hash.rb
+- lib/ids_of_all_dependencies.rb
+- lib/model.rb
+- lib/models/abuse.rb
+- lib/models/annotation.rb
 - lib/models/branch.rb
+- lib/models/broadcast.rb
 - lib/models/build.rb
 - lib/models/commit.rb
 - lib/models/cron.rb
+- lib/models/email.rb
+- lib/models/invoice.rb
 - lib/models/job.rb
 - lib/models/log.rb
-- lib/models/model.rb
+- lib/models/membership.rb
+- lib/models/message.rb
 - lib/models/organization.rb
+- lib/models/owner_group.rb
+- lib/models/permission.rb
 - lib/models/pull_request.rb
+- lib/models/queueable_job.rb
 - lib/models/repository.rb
 - lib/models/request.rb
 - lib/models/ssl_key.rb
 - lib/models/stage.rb
+- lib/models/star.rb
+- lib/models/subscription.rb
 - lib/models/tag.rb
+- lib/models/token.rb
+- lib/models/trial.rb
+- lib/models/trial_allowance.rb
 - lib/models/user.rb
+- lib/models/user_beta_feature.rb
+- lib/nullify_dependencies.rb
 - lib/travis-backup.rb
 - log/.keep
 - package.json

data/lib/backup/remove_old.rb

@@ -1,204 +0,0 @@
-# frozen_string_literal: true
-
-class Backup
-  class RemoveOld
-    attr_reader :config
-
-    def initialize(config, dry_run_reporter=nil)
-      @config = config
-      @dry_run_reporter = dry_run_reporter
-    end
-
-    def dry_run_report
-      @dry_run_reporter.report
-    end
-
-    def run(args={})
-      user_id = args[:user_id] || @config.user_id
-      repo_id = args[:repo_id] || @config.repo_id
-      org_id = args[:org_id] || @config.org_id
-
-      if user_id
-        process_repos_for_owner(user_id, 'User')
-      elsif org_id
-        process_repos_for_owner(org_id, 'Organization')
-      elsif repo_id
-        process_repo_with_id(repo_id)
-      else
-        process_all_repos
-      end  
-    end
-
-    def process_repos_for_owner(owner_id, owner_type)
-      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-        process_repo(repository)
-      end
-    end
-  
-    def process_repo_with_id(repo_id)
-      process_repo(Repository.find(repo_id))
-    end
-  
-    def process_all_repos
-      Repository.order(:id).each do |repository|
-        process_repo(repository)
-      end
-    end
-  
-    def process_repo(repository)
-      process_repo_builds(repository)
-      process_repo_requests(repository)
-    end
-  
-    def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      threshold = @config.threshold.to_i.months.ago.to_datetime
-      current_build_id = repository.current_build_id || -1
-      repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-                .in_batches(of: @config.limit.to_i).map do |builds_batch|
-        if @config.if_backup
-          file_prefix = "repository_#{repository.id}"
-          save_and_destroy_builds_batch(builds_batch, file_prefix)
-        else
-          destroy_builds_batch(builds_batch)
-        end
-      end.compact.reduce(&:&)
-    end
-  
-    def process_repo_requests(repository)
-      threshold = @config.threshold.to_i.months.ago.to_datetime
-      repository.requests.where('created_at < ?', threshold)
-                .in_batches(of: @config.limit.to_i).map do |requests_batch|
-        @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
-      end.compact
-    end
-  
-    private
-  
-    def save_and_destroy_builds_batch(builds_batch, file_prefix)
-      builds_export = builds_batch.map(&:attributes)
-  
-      dependencies_saved = builds_batch.map do |build|
-        save_build_jobs_and_logs(build, file_prefix)
-      end.reduce(&:&)
-  
-      if dependencies_saved
-        file_name = "#{file_prefix}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
-        pretty_json = JSON.pretty_generate(builds_export)
-        save_file(file_name, pretty_json) ? destroy_builds_batch(builds_batch) : false
-      else
-        false
-      end
-    end
-  
-    def save_build_jobs_and_logs(build, file_prefix)
-      build.jobs.in_batches(of: @config.limit.to_i).map do |jobs_batch|
-        file_prefix = "#{file_prefix}_build_#{build.id}"
-        save_jobs_batch(jobs_batch, file_prefix)
-      end.compact.reduce(&:&)
-    end
-  
-    def save_jobs_batch(jobs_batch, file_prefix)
-      jobs_export = jobs_batch.map(&:attributes)
-  
-      logs_saved = jobs_batch.map do |job|
-        save_job_logs(job, file_prefix)
-      end.reduce(&:&)
-  
-      if logs_saved
-        file_name = "#{file_prefix}_jobs_#{jobs_batch.first.id}-#{jobs_batch.last.id}.json"
-        pretty_json = JSON.pretty_generate(jobs_export)
-        save_file(file_name, pretty_json)
-      else
-        false
-      end
-    end
-  
-    def save_job_logs(job, file_prefix)
-      job.logs.in_batches(of: @config.limit.to_i).map do |logs_batch|
-        file_prefix = "#{file_prefix}_job_#{job.id}"
-        save_logs_batch(logs_batch, file_prefix)
-      end.compact.reduce(&:&)
-    end
-  
-    def save_logs_batch(logs_batch, file_prefix)
-      logs_export = logs_batch.map(&:attributes)
-      file_name = "#{file_prefix}_logs_#{logs_batch.first.id}-#{logs_batch.last.id}.json"
-      pretty_json = JSON.pretty_generate(logs_export)
-      save_file(file_name, pretty_json)
-    end
-  
-    def destroy_builds_batch(builds_batch)
-      return destroy_builds_batch_dry(builds_batch) if @config.dry_run
-  
-      builds_batch.each(&:destroy)
-    end
-  
-    def destroy_builds_batch_dry(builds_batch)
-      @dry_run_reporter.add_to_report(:builds, *builds_batch.map(&:id))
-  
-      jobs_ids = builds_batch.map do |build|
-        build.jobs.map(&:id) || []
-      end.flatten
-  
-      @dry_run_reporter.add_to_report(:jobs, *jobs_ids)
-  
-      logs_ids = builds_batch.map do |build|
-        build.jobs.map do |job|
-          job.logs.map(&:id) || []
-        end.flatten || []
-      end.flatten
-  
-      @dry_run_reporter.add_to_report(:logs, *logs_ids)
-    end
-  
-    def save_and_destroy_requests_batch(requests_batch, repository)
-      requests_export = export_requests(requests_batch)
-      file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
-      pretty_json = JSON.pretty_generate(requests_export)
-      if save_file(file_name, pretty_json)
-        destroy_requests_batch(requests_batch)
-      end
-      requests_export
-    end
-  
-    def destroy_requests_batch(requests_batch)
-      return destroy_requests_batch_dry(requests_batch) if @config.dry_run
-  
-      requests_batch.each(&:destroy)
-    end
-  
-    def destroy_requests_batch_dry(requests_batch)
-      @dry_run_reporter.add_to_report(:requests, *requests_batch.map(&:id))
-    end
-  
-    def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
-      return true if @config.dry_run
-  
-      saved = false
-      begin
-        unless File.directory?(@config.files_location)
-          FileUtils.mkdir_p(@config.files_location)
-        end
-  
-        File.open(file_path(file_name), 'w') do |file|
-          file.write(content)
-          file.close
-          saved = true
-        end
-      rescue => e
-        print "Failed to save #{file_name}, error: #{e.inspect}\n"
-      end
-      saved
-    end
-  
-    def file_path(file_name)
-      "#{@config.files_location}/#{file_name}"
-    end
-  
-    def export_requests(requests)
-      requests.map do |request|
-        request.attributes
-      end
-    end    
-  end
-end

data/lib/models/model.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_record'
-
-# Model class
-class Model < ActiveRecord::Base
-  self.abstract_class = true
-end