travis-backup 0.0.3 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dca7a53c9e3689c5f0988d074296449769a106b86ed6ddc26c6a8b7c0cba15b2
-  data.tar.gz: 25765ff21af6e3bbba2fbf5a971d678acd6447b59c54fc685ddaa0912e84f73a
+  metadata.gz: 4165ff1d9cbf20dae8d4e20f8a481b6ae3cc64a93f82fadae2d1159853bcdd31
+  data.tar.gz: e87df3c1a1dc808308abd0ff46d24fb4479a8cba9407cca38a82948e1e350f8a
 SHA512:
-  metadata.gz: bcc37fc6219ac84eda44f953879952567eb33981a9a2436b5922acefa515994262ab174e25e8fe9660153e73e3d55838eaa8fd375dc8f79cefff02f574f8e516
-  data.tar.gz: 377cef9823245bd5153bc291e6fb078c723544986e6fd85a79e50b64d8cc5110c1331ac8247a40b158022633672ad93e3a85719d2b70f4659e152913f0b4b729
+  metadata.gz: bef72b11c7596d2fa35f618afa684056e4b7a0c9c94217242683c56b62e4e8f8873c23ee6a5ee01ab935da5126cea0fa9894ef06510d2f926471a4004f92e5a3
+  data.tar.gz: f196c2233ff24dd151d48a7d294a648ea9fee1db24ceb4ee325ccb3efb00675bb809d639403181985041120cc4aadd420a7398aa899228e94562e9acb0b9771b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    travis-backup (0.0.3)
+    travis-backup (0.1.3)
       activerecord
       bootsnap
       pg

data/README.md

@@ -1,7 +1,6 @@
 # README
 
-*travis-backup* is an application that removes builds and their corresponding jobs
-and exports them (optionally) to json files.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data.
 
 ### Installation and run
 
@@ -27,6 +26,9 @@ All arguments:
   -u, --user_id ID           # run only for given user
   -o, --org_id ID            # run only for given organization
   -r, --repo_id ID           # run only for given repository
+  --move_logs                # run in move logs mode - move all logs to database at destination_db_url URL
+  --destination_db_url URL   # URL for moving logs to
+  --remove_orphans           # run in remove orphans mode
 ```
 
 Or inside your app:
@@ -54,9 +56,17 @@ backup.run(org_id: 1)
 backup.run(repo_id: 1)
 ```
 
-### Configuration
+#### Special modes
 
-Despite of command line arguments, one of the ways you can configure your export is a file `config/settinigs.yml` that you can place in your app's main directory. The gem expects properties in the following format:
+Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). When you run gem in this mode no files are created and no other tables are being touched.
+
+Using `--remove_orphans` flag you can remove all orphaned data from tables. When you run gem in this mode no files are created.
+
+Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with `--move_logs` or `--remove_orphans`.
+
+### Configuration options
+
+Despite of command line arguments, one of the ways you can configure your export is a file `config/settings.yml` that you can place in your app's main directory. The gem expects properties in the following format:
 
 ```
 backup:
@@ -65,15 +75,19 @@ backup:
   limit: 1000               # builds limit for one backup file
   threshold: 6              # number of months from now - data younger than this time won't be backuped
   files_location: './dump'  # path of the folder in which backup files will be placed
-  user_id                   # run only for given user
-  org_id                    # run only for given organization
-  repo_id                   # run only for given repository
+  user_id: 1                # run only for given user
+  org_id: 1                 # run only for given organization
+  repo_id: 1                # run only for given repository
+  move_logs: false          # run in move logs mode - move all logs to database at destination_db_url URL
+  remove_orphans: false     # run in remove orphans mode
 ```
 
-You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`.
+You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`.
 
 You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema.
 
+For `move_logs` mode you need also to specify a destination database. You can set it also in `config/database.yml` file, in `destination` subsection, setting the `destination_db_url` hash argument while creating `Backup` instance or using the `BACKUP_DESTINATION_DB_URL` env var. Your destination database should be consistent with the Travis 3.0 database schema.
+
 ### How to run the test suite
 
 You can run the test after cloning this repository. Next you should call
@@ -88,9 +102,9 @@ and
 bundle exec rspec
 ```
 
-To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`.
+To make tests working properly you should also ensure database connection strings for empty test databases. You can set them as `DATABASE_URL` and `BACKUP_DESTINATION_DB_URL` environment variables or in `config/database.yml`.
 
-**Warning: this database will be cleaned during tests, so ensure that it includes no important data.**
+**Warning: these databases will be cleaned during tests, so ensure that they include no important data.**
 
 ### Ruby version
 

data/lib/config.rb

@@ -12,6 +12,7 @@ class Config
     :repo_id,
     :org_id,
     :move_logs,
+    :remove_orphans,
     :destination_db_url
 
   def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
@@ -66,19 +67,19 @@ class Config
     @user_id = first_not_nil(
       args[:user_id],
       argv_opts[:user_id],
-      ENV['USER_ID'],
+      ENV['BACKUP_USER_ID'],
       config.dig('backup', 'user_id')
     )
     @repo_id = first_not_nil(
       args[:repo_id],
       argv_opts[:repo_id],
-      ENV['REPO_ID'],
+      ENV['BACKUP_REPO_ID'],
       config.dig('backup', 'repo_id')
     )
     @org_id = first_not_nil(
       args[:org_id],
       argv_opts[:org_id],
-      ENV['ORG_ID'],
+      ENV['BACKUP_ORG_ID'],
       config.dig('backup', 'org_id')
     )
     @move_logs = first_not_nil(
@@ -88,6 +89,13 @@ class Config
       config.dig('backup', 'move_logs'),
       false
     )
+    @remove_orphans = first_not_nil(
+      args[:remove_orphans],
+      argv_opts[:remove_orphans],
+      ENV['BACKUP_REMOVE_ORPHANS'],
+      config.dig('backup', 'remove_orphans'),
+      false
+    )
     @destination_db_url = first_not_nil(
       args[:destination_db_url],
       argv_opts[:destination_db_url],
@@ -97,7 +105,7 @@ class Config
   end
 
   def check_values
-    if !@move_logs && !@threshold
+    if !@move_logs && !@remove_orphans && !@threshold
       message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
         "Threshold defines number of months from now.")
       abort message
@@ -138,6 +146,7 @@ class Config
       opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
       opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
       opt.on('--move_logs') { |o| options[:move_logs] = o }
+      opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
       opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
     end.parse!
 

data/lib/models/branch.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Branch < Model
+  self.table_name = 'branches'
+end

data/lib/models/build.rb

@@ -8,6 +8,7 @@ require 'models/repository'
 class Build < Model
   belongs_to :repository
   has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job'
+  has_one    :repo_for_that_this_build_is_current, foreign_key: :current_build_id, dependent: :destroy, class_name: 'Repository'
 
   self.table_name = 'builds'
 end

data/lib/models/commit.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Commit < Model
+  self.table_name = 'commits'
+end

data/lib/models/cron.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Cron < Model
+  self.table_name = 'crons'
+end

data/lib/models/pull_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class PullRequest < Model
+  self.table_name = 'pull_requests'
+end

data/lib/models/repository.rb

@@ -2,10 +2,12 @@
 
 require 'models/model'
 require 'models/build'
+require 'models/request'
 
 # Repository model
 class Repository < Model
-  has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build'
+  has_many :builds, -> { order('id') }, foreign_key: :repository_id, class_name: 'Build'
+  has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request'
 
   self.table_name = 'repositories'
 end

data/lib/models/request.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'models/model'
+require 'models/repository'
+
+class Request < Model
+  belongs_to :repository
+
+  self.table_name = 'requests'
+end

data/lib/models/ssl_key.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class SslKey < Model
+  self.table_name = 'ssl_keys'
+end

data/lib/models/stage.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Stage < Model
+  self.table_name = 'stages'
+end

data/lib/models/tag.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Tag < Model
+  self.table_name = 'tags'
+end

data/lib/travis-backup.rb

@@ -5,17 +5,25 @@ require 'active_support/time'
 require 'config'
 require 'models/repository'
 require 'models/log'
+require 'models/branch'
+require 'models/tag'
+require 'models/commit'
+require 'models/cron'
+require 'models/pull_request'
+require 'models/ssl_key'
+require 'models/request'
+require 'models/stage'
 
 # main travis-backup class
 class Backup
   attr_accessor :config
-  attr_reader :dry_run_removed
+  attr_reader :dry_run_report
 
   def initialize(config_args={})
     @config = Config.new(config_args)
 
     if @config.dry_run
-      @dry_run_removed = {builds: [], jobs: [], logs: []}
+      @dry_run_report = {builds: [], jobs: [], logs: [], requests: []}
     end
 
     connect_db
@@ -26,8 +34,6 @@ class Backup
   end
 
   def run(args={})
-    return move_logs if @config.move_logs
-
     user_id = args[:user_id] || @config.user_id
     repo_id = args[:repo_id] || @config.repo_id
     org_id = args[:org_id] || @config.org_id
@@ -38,32 +44,80 @@ class Backup
     elsif org_id
       owner_id = org_id
       owner_type = 'Organization'
-    elsif repo_id
-      repo_id = repo_id
     end
 
-    if owner_id
-      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-        process_repo(repository)
-      end
+    if @config.move_logs
+      move_logs
+    elsif @config.remove_orphans
+      remove_orphans
+    elsif owner_id
+      process_repos_for_owner(owner_id, owner_type)
     elsif repo_id
-      repository = Repository.find(repo_id)
+      process_repo_with_id(repo_id)
+    else
+      process_all_repos
+    end
+
+    print_dry_run_report if @config.dry_run
+  end
+
+  def process_repos_for_owner(owner_id, owner_type)
+    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
       process_repo(repository)
+    end
+  end
+
+  def process_repo_with_id(repo_id)
+    process_repo(Repository.find(repo_id))
+  end
+
+  def process_all_repos
+    Repository.order(:id).each do |repository|
+      process_repo(repository)
+    end
+  end
+
+  def print_dry_run_report
+    if @dry_run_report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
     else
-      Repository.order(:id).each do |repository|
-        process_repo(repository)
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @dry_run_report.to_a.map(&:first).each do |symbol|
+        print_dry_run_report_line(symbol)
       end
     end
+  end
 
-    if @config.dry_run
-      puts 'Dry run active. The following data would be removed in normal mode:'
-      puts " - builds: #{@dry_run_removed[:builds].to_json}"
-      puts " - jobs: #{@dry_run_removed[:jobs].to_json}"
-      puts " - logs: #{@dry_run_removed[:logs].to_json}"
-    end
+  def print_dry_run_report_line(symbol)
+    puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any?
+  end
+
+  def process_repo(repository)
+    process_repo_builds(repository)
+    process_repo_requests(repository)
+  end
+
+  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    current_build_id = repository.current_build_id || -1
+    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
+              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
+      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
+    end.compact
+  end
+
+  def process_repo_requests(repository)
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    repository.requests.where('created_at < ?', threshold)
+              .in_groups_of(@config.limit.to_i, false).map do |requests_batch|
+      @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
+    end.compact
   end
 
   def move_logs
+    return move_logs_dry if config.dry_run
+
     connect_db(@config.database_url)
     Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
       log_hashes = logs_batch.as_json
@@ -80,41 +134,86 @@ class Backup
     end
   end
 
-  def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    current_build_id = repository.current_build_id || -1
-    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch)
-    end.compact
+  def move_logs_dry
+    dry_run_report[:logs].concat(Log.order(:id).map(&:id))
+  end
+
+  def remove_orphans
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id')
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id')
+    remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id')
+    remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id')
+    remove_orphans_for_table(Build, 'builds', 'requests', 'request_id')
+    remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id')
+    remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id')
+    remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id')
+    remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id')
+    remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id')
+    remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id')
+    remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id')
+    remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id')
+    remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id')
+    remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id')
+    remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id')
+    remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id')
+    remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id')
+    remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id')
+    remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id')
+    remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id')
+    remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id')
+    remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id')
+    remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id')
+  end
+
+  def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
+    for_delete = model_class.find_by_sql(%{
+      select a.*
+      from #{table_a_name} a
+      left join #{table_b_name} b
+      on a.#{fk_name} = b.id
+      where
+        a.#{fk_name} is not null
+        and b.id is null;
+    })
+
+    if config.dry_run
+      key = table_a_name.to_sym
+      dry_run_report[key] = [] if dry_run_report[key].nil?
+      dry_run_report[key].concat(for_delete.map(&:id))
+      dry_run_report[key].uniq!
+    else
+      model_class.where(id: for_delete.map(&:id)).destroy_all
+    end
   end
 
   private
 
-  def save_and_destroy_batch(builds_batch, repository)
+  def save_and_destroy_builds_batch(builds_batch, repository)
     builds_export = export_builds(builds_batch)
     file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
     pretty_json = JSON.pretty_generate(builds_export)
     if save_file(file_name, pretty_json)
-      destroy_batch(builds_batch)
+      destroy_builds_batch(builds_batch)
     end
     builds_export
   end
 
-  def destroy_batch(builds_batch)
-    return destroy_batch_dry(builds_batch) if @config.dry_run
+  def destroy_builds_batch(builds_batch)
+    return destroy_builds_batch_dry(builds_batch) if @config.dry_run
 
     builds_batch.each(&:destroy)
   end
 
-  def destroy_batch_dry(builds_batch)
-    @dry_run_removed[:builds].concat(builds_batch.map(&:id))
+  def destroy_builds_batch_dry(builds_batch)
+    @dry_run_report[:builds].concat(builds_batch.map(&:id))
 
     jobs = builds_batch.map do |build|
       build.jobs.map(&:id) || []
     end.flatten
 
-    @dry_run_removed[:jobs].concat(jobs)
+    @dry_run_report[:jobs].concat(jobs)
 
     logs = builds_batch.map do |build|
       build.jobs.map do |job|
@@ -122,7 +221,27 @@ class Backup
       end.flatten || []
     end.flatten
 
-    @dry_run_removed[:logs].concat(logs)
+    @dry_run_report[:logs].concat(logs)
+  end
+
+  def save_and_destroy_requests_batch(requests_batch, repository)
+    requests_export = export_requests(requests_batch)
+    file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+    pretty_json = JSON.pretty_generate(requests_export)
+    if save_file(file_name, pretty_json)
+      destroy_requests_batch(requests_batch)
+    end
+    requests_export
+  end
+
+  def destroy_requests_batch(requests_batch)
+    return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+
+    requests_batch.each(&:destroy)
+  end
+
+  def destroy_requests_batch_dry(requests_batch)
+    @dry_run_report[:requests].concat(requests_batch.map(&:id))
   end
 
   def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
@@ -169,9 +288,13 @@ class Backup
 
   def export_logs(logs)
     logs.map do |log|
-      log_export = log.attributes
+      log.attributes
+    end
+  end
 
-      log_export
+  def export_requests(requests)
+    requests.map do |request|
+      request.attributes
     end
   end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.0.3'
+  s.version               = '0.1.3'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.3
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -248,12 +248,20 @@ files:
 - db/schema.sql
 - dump/.keep
 - lib/config.rb
+- lib/models/branch.rb
 - lib/models/build.rb
+- lib/models/commit.rb
+- lib/models/cron.rb
 - lib/models/job.rb
 - lib/models/log.rb
 - lib/models/model.rb
 - lib/models/organization.rb
+- lib/models/pull_request.rb
 - lib/models/repository.rb
+- lib/models/request.rb
+- lib/models/ssl_key.rb
+- lib/models/stage.rb
+- lib/models/tag.rb
 - lib/models/user.rb
 - lib/travis-backup.rb
 - log/.keep