travis-backup 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dca7a53c9e3689c5f0988d074296449769a106b86ed6ddc26c6a8b7c0cba15b2
-  data.tar.gz: 25765ff21af6e3bbba2fbf5a971d678acd6447b59c54fc685ddaa0912e84f73a
+  metadata.gz: da5ede447a9a2a1b215d03d8582f62e8c4573e69e3337da91afaaffef07323f3
+  data.tar.gz: 01bc3c92f48f093ae3dd4ff23c07a0f2e9c5516b79c05279700d82570b0a2d40
 SHA512:
-  metadata.gz: bcc37fc6219ac84eda44f953879952567eb33981a9a2436b5922acefa515994262ab174e25e8fe9660153e73e3d55838eaa8fd375dc8f79cefff02f574f8e516
-  data.tar.gz: 377cef9823245bd5153bc291e6fb078c723544986e6fd85a79e50b64d8cc5110c1331ac8247a40b158022633672ad93e3a85719d2b70f4659e152913f0b4b729
+  metadata.gz: 775e8b95e35be972da5194d703d33645bfe21198d91f28efe2a417214d74e4fc71fee5a9c6d7a715ae2819b8d12a0b0bad691bca44b507782acd1dbf48c43d16
+  data.tar.gz: 289107dfc9d91b3cc43479c1a4973fe0fec4db91a519dd9cd8be321c9ce56f65e2170464ae3c88492385af5609d0ea52bfbe9d7510f1a44bcffa8f1dfea81b63

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    travis-backup (0.0.3)
+    travis-backup (0.1.0)
       activerecord
       bootsnap
       pg

data/README.md

@@ -1,7 +1,6 @@
 # README
 
-*travis-backup* is an application that removes builds and their corresponding jobs
-and exports them (optionally) to json files.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database.
 
 ### Installation and run
 
@@ -27,6 +26,9 @@ All arguments:
   -u, --user_id ID           # run only for given user
   -o, --org_id ID            # run only for given organization
   -r, --repo_id ID           # run only for given repository
+  --move_logs                # run in move logs mode - move all logs to database at destination_db_url URL
+  --destination_db_url URL   # URL for moving logs to
+  --remove_orphans           # run in remove orphans mode
 ```
 
 Or inside your app:
@@ -54,9 +56,17 @@ backup.run(org_id: 1)
 backup.run(repo_id: 1)
 ```
 
-### Configuration
+#### Special modes
 
-Despite of command line arguments, one of the ways you can configure your export is a file `config/settinigs.yml` that you can place in your app's main directory. The gem expects properties in the following format:
+Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). When you run gem in this mode no files are created and no other tables are being touched.
+
+Using `--remove_orphans` flag you can remove all orphaned data from tables. When you run gem in this mode no files are created.
+
+Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with `--move_logs` or `--remove_orphans`.
+
+### Configuration options
+
+Despite of command line arguments, one of the ways you can configure your export is a file `config/settings.yml` that you can place in your app's main directory. The gem expects properties in the following format:
 
 ```
 backup:
@@ -88,9 +98,9 @@ and
 bundle exec rspec
 ```
 
-To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`.
+To make tests working properly you should also ensure database connection strings for empty test databases. You can set them as `DATABASE_URL` and `BACKUP_DESTINATION_DB_URL` environment variables or in `config/database.yml`.
 
-**Warning: this database will be cleaned during tests, so ensure that it includes no important data.**
+**Warning: these databases will be cleaned during tests, so ensure that they include no important data.**
 
 ### Ruby version
 

data/lib/config.rb

@@ -12,6 +12,7 @@ class Config
     :repo_id,
     :org_id,
     :move_logs,
+    :remove_orphans,
     :destination_db_url
 
   def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
@@ -88,6 +89,13 @@ class Config
       config.dig('backup', 'move_logs'),
       false
     )
+    @remove_orphans = first_not_nil(
+      args[:remove_orphans],
+      argv_opts[:remove_orphans],
+      ENV['BACKUP_REMOVE_ORPHANS'],
+      config.dig('backup', 'remove_orphans'),
+      false
+    )
     @destination_db_url = first_not_nil(
       args[:destination_db_url],
       argv_opts[:destination_db_url],
@@ -97,7 +105,7 @@ class Config
   end
 
   def check_values
-    if !@move_logs && !@threshold
+    if !@move_logs && !@remove_orphans && !@threshold
       message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
         "Threshold defines number of months from now.")
       abort message
@@ -138,6 +146,7 @@ class Config
       opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
       opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
       opt.on('--move_logs') { |o| options[:move_logs] = o }
+      opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
       opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
     end.parse!
 

data/lib/models/branch.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Branch < Model
+  self.table_name = 'branches'
+end

data/lib/models/commit.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Commit < Model
+  self.table_name = 'commits'
+end

data/lib/models/cron.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Cron < Model
+  self.table_name = 'crons'
+end

data/lib/models/pull_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class PullRequest < Model
+  self.table_name = 'pull_requests'
+end

data/lib/models/repository.rb

@@ -2,10 +2,12 @@
 
 require 'models/model'
 require 'models/build'
+require 'models/request'
 
 # Repository model
 class Repository < Model
   has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build'
+  has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request'
 
   self.table_name = 'repositories'
 end

data/lib/models/request.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'models/model'
+require 'models/repository'
+
+class Request < Model
+  belongs_to :repository
+
+  self.table_name = 'requests'
+end

data/lib/models/ssl_key.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class SslKey < Model
+  self.table_name = 'ssl_keys'
+end

data/lib/models/stage.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Stage < Model
+  self.table_name = 'stages'
+end

data/lib/models/tag.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Tag < Model
+  self.table_name = 'tags'
+end

data/lib/travis-backup.rb

@@ -5,17 +5,25 @@ require 'active_support/time'
 require 'config'
 require 'models/repository'
 require 'models/log'
+require 'models/branch'
+require 'models/tag'
+require 'models/commit'
+require 'models/cron'
+require 'models/pull_request'
+require 'models/ssl_key'
+require 'models/request'
+require 'models/stage'
 
 # main travis-backup class
 class Backup
   attr_accessor :config
-  attr_reader :dry_run_removed
+  attr_reader :dry_run_report
 
   def initialize(config_args={})
     @config = Config.new(config_args)
 
     if @config.dry_run
-      @dry_run_removed = {builds: [], jobs: [], logs: []}
+      @dry_run_report = {builds: [], jobs: [], logs: [], requests: []}
     end
 
     connect_db
@@ -26,8 +34,6 @@ class Backup
   end
 
   def run(args={})
-    return move_logs if @config.move_logs
-
     user_id = args[:user_id] || @config.user_id
     repo_id = args[:repo_id] || @config.repo_id
     org_id = args[:org_id] || @config.org_id
@@ -38,32 +44,80 @@ class Backup
     elsif org_id
       owner_id = org_id
       owner_type = 'Organization'
-    elsif repo_id
-      repo_id = repo_id
     end
 
-    if owner_id
-      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-        process_repo(repository)
-      end
+    if @config.move_logs
+      move_logs
+    elsif @config.remove_orphans
+      remove_orphans
+    elsif owner_id
+      process_repos_for_owner(owner_id, owner_type)
     elsif repo_id
-      repository = Repository.find(repo_id)
+      process_repo_with_id(repo_id)
+    else
+      process_all_repos
+    end
+
+    print_dry_run_report if @config.dry_run
+  end
+
+  def process_repos_for_owner(owner_id, owner_type)
+    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
       process_repo(repository)
+    end
+  end
+
+  def process_repo_with_id(repo_id)
+    process_repo(Repository.find(repo_id))
+  end
+
+  def process_all_repos
+    Repository.order(:id).each do |repository|
+      process_repo(repository)
+    end
+  end
+
+  def print_dry_run_report
+    if @dry_run_report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
     else
-      Repository.order(:id).each do |repository|
-        process_repo(repository)
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @dry_run_report.to_a.map(&:first).each do |symbol|
+        print_dry_run_report_line(symbol)
       end
     end
+  end
 
-    if @config.dry_run
-      puts 'Dry run active. The following data would be removed in normal mode:'
-      puts " - builds: #{@dry_run_removed[:builds].to_json}"
-      puts " - jobs: #{@dry_run_removed[:jobs].to_json}"
-      puts " - logs: #{@dry_run_removed[:logs].to_json}"
-    end
+  def print_dry_run_report_line(symbol)
+    puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any?
+  end
+
+  def process_repo(repository)
+    process_repo_builds(repository)
+    process_repo_requests(repository)
+  end
+
+  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    current_build_id = repository.current_build_id || -1
+    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
+              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
+      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
+    end.compact
+  end
+
+  def process_repo_requests(repository)
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    repository.requests.where('created_at < ?', threshold)
+              .in_groups_of(@config.limit.to_i, false).map do |requests_batch|
+      @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
+    end.compact
   end
 
   def move_logs
+    return move_logs_dry if config.dry_run
+
     connect_db(@config.database_url)
     Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
       log_hashes = logs_batch.as_json
@@ -80,41 +134,86 @@ class Backup
     end
   end
 
-  def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    current_build_id = repository.current_build_id || -1
-    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch)
-    end.compact
+  def move_logs_dry
+    dry_run_report[:logs].concat(Log.order(:id).map(&:id))
+  end
+
+  def remove_orphans
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id')
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id')
+    remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id')
+    remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id')
+    remove_orphans_for_table(Build, 'builds', 'requests', 'request_id')
+    remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id')
+    remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id')
+    remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id')
+    remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id')
+    remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id')
+    remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id')
+    remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id')
+    remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id')
+    remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id')
+    remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id')
+    remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id')
+    remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id')
+    remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id')
+    remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id')
+    remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id')
+    remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id')
+    remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id')
+    remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id')
+    remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id')
+  end
+
+  def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
+    for_delete = model_class.find_by_sql(%{
+      select a.*
+      from #{table_a_name} a
+      left join #{table_b_name} b
+      on a.#{fk_name} = b.id
+      where
+        a.#{fk_name} is not null
+        and b.id is null;
+    })
+
+    if config.dry_run
+      key = table_a_name.to_sym
+      dry_run_report[key] = [] if dry_run_report[key].nil?
+      dry_run_report[key].concat(for_delete.map(&:id))
+      dry_run_report[key].uniq!
+    else
+      model_class.where(id: for_delete.map(&:id)).delete_all
+    end
   end
 
   private
 
-  def save_and_destroy_batch(builds_batch, repository)
+  def save_and_destroy_builds_batch(builds_batch, repository)
     builds_export = export_builds(builds_batch)
     file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
     pretty_json = JSON.pretty_generate(builds_export)
     if save_file(file_name, pretty_json)
-      destroy_batch(builds_batch)
+      destroy_builds_batch(builds_batch)
     end
     builds_export
   end
 
-  def destroy_batch(builds_batch)
-    return destroy_batch_dry(builds_batch) if @config.dry_run
+  def destroy_builds_batch(builds_batch)
+    return destroy_builds_batch_dry(builds_batch) if @config.dry_run
 
     builds_batch.each(&:destroy)
   end
 
-  def destroy_batch_dry(builds_batch)
-    @dry_run_removed[:builds].concat(builds_batch.map(&:id))
+  def destroy_builds_batch_dry(builds_batch)
+    @dry_run_report[:builds].concat(builds_batch.map(&:id))
 
     jobs = builds_batch.map do |build|
       build.jobs.map(&:id) || []
     end.flatten
 
-    @dry_run_removed[:jobs].concat(jobs)
+    @dry_run_report[:jobs].concat(jobs)
 
     logs = builds_batch.map do |build|
       build.jobs.map do |job|
@@ -122,7 +221,27 @@ class Backup
       end.flatten || []
     end.flatten
 
-    @dry_run_removed[:logs].concat(logs)
+    @dry_run_report[:logs].concat(logs)
+  end
+
+  def save_and_destroy_requests_batch(requests_batch, repository)
+    requests_export = export_requests(requests_batch)
+    file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+    pretty_json = JSON.pretty_generate(requests_export)
+    if save_file(file_name, pretty_json)
+      destroy_requests_batch(requests_batch)
+    end
+    requests_export
+  end
+
+  def destroy_requests_batch(requests_batch)
+    return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+
+    requests_batch.each(&:destroy)
+  end
+
+  def destroy_requests_batch_dry(requests_batch)
+    @dry_run_report[:requests].concat(requests_batch.map(&:id))
   end
 
   def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
@@ -169,9 +288,13 @@ class Backup
 
   def export_logs(logs)
     logs.map do |log|
-      log_export = log.attributes
+      log.attributes
+    end
+  end
 
-      log_export
+  def export_requests(requests)
+    requests.map do |request|
+      request.attributes
     end
   end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.0.3'
+  s.version               = '0.1.0'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -248,12 +248,20 @@ files:
 - db/schema.sql
 - dump/.keep
 - lib/config.rb
+- lib/models/branch.rb
 - lib/models/build.rb
+- lib/models/commit.rb
+- lib/models/cron.rb
 - lib/models/job.rb
 - lib/models/log.rb
 - lib/models/model.rb
 - lib/models/organization.rb
+- lib/models/pull_request.rb
 - lib/models/repository.rb
+- lib/models/request.rb
+- lib/models/ssl_key.rb
+- lib/models/stage.rb
+- lib/models/tag.rb
 - lib/models/user.rb
 - lib/travis-backup.rb
 - log/.keep