travis-backup 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b88fafbc38de9343ca5ead6600c182858c39b14210a60f05081543bf74c8964
-  data.tar.gz: d45a8734295f8dfb01675e333d99c6e19b441fd2290be042ced8b0e2d3e161c8
+  metadata.gz: f8b5825713610ce11c333d20573340404da43e46afadad8a8149b35194aa371e
+  data.tar.gz: acc50d6c047c57af32368a5fcd634e130a9ccab611222f1bf8b235be26e474fb
 SHA512:
-  metadata.gz: 879009e7cbffc3be645bc12746ce25598b9d013ca81e0aa18370644c6c5bfba3d300d67fcd297e36fc0f941c41a0250cbd50fb3426211ae655e16c0f006b2a56
-  data.tar.gz: 316429ee0dd693f3ba737abbc341f0afef919c4ff85d677fc2652dc71f42b6ba6a76580526f0f42f8bff8ed9fc28fbd0743f3157f559209f19844899ee13b421
+  metadata.gz: 14ac9ae0fdc86e32e68dc396a8ad02cc0adf988a6614146ba46c2353a290d4e57d33e5ead26a5cb7f8f8d6c4654c1833fb71af0ce9725d0431e5698d9c984adc
+  data.tar.gz: b07f0529a93a964f3280daaad33ceaff21c547438d93709c256b905cc8325e5cdfaf5d74204a0a31cd0fa497e43958f789dd66b600d1c5e131a9c74f50003222

data/.travis.yml

@@ -30,4 +30,7 @@ before_script:
   - psql --version
   - psql -c 'CREATE DATABASE travis_test;' -U postgres
   - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
-  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test
+  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test
+  - psql -c 'CREATE DATABASE travis_test_destination;' -U postgres
+  - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
+  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test_destination

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    travis-backup (0.0.1)
+    travis-backup (0.1.1)
       activerecord
       bootsnap
       pg

data/README.md

@@ -1,7 +1,6 @@
 # README
 
-*travis-backup* is an application that removes builds and their corresponding jobs
-and exports them (optionally) to json files.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data.
 
 ### Installation and run
 
@@ -9,15 +8,33 @@ You can install the gem using
 
 `gem install travis-backup`
 
-Next you can run it in your app like
+Next you can run it like:
 
 ```
-require 'travis-backup'
+travis_backup 'postgres://user:pass@localhost:5432/my_db' --threshold 6
+```
 
-backup = Backup.new
-backup.run
+All arguments:
+
+```
+  first argument, no flag    # database url
+  -b, --backup               # when not present, removes data without saving it to file
+  -d, --dry_run              # only prints in console what data will be backuped and deleted
+  -l, --limit LIMIT          # builds limit for one backup file
+  -t, --threshold MONTHS     # number of months from now - data younger than this time won't be backuped
+  -f, --files_location PATH  # path of the folder in which backup files will be placed
+  -u, --user_id ID           # run only for given user
+  -o, --org_id ID            # run only for given organization
+  -r, --repo_id ID           # run only for given repository
+  --move_logs                # run in move logs mode - move all logs to database at destination_db_url URL
+  --destination_db_url URL   # URL for moving logs to
+  --remove_orphans           # run in remove orphans mode
+```
 
-# or with configs:
+Or inside your app:
+
+```
+require 'travis-backup'
 
 backup = Backup.new(
   if_backup: true,
@@ -32,7 +49,6 @@ backup.run
 You can also run backup only for given user, organisation or repository:
 
 ```
-backup = Backup.new
 backup.run(user_id: 1)
 # or
 backup.run(org_id: 1)
@@ -40,9 +56,17 @@ backup.run(org_id: 1)
 backup.run(repo_id: 1)
 ```
 
-### Configuration
+#### Special modes
+
+Using `--move_logs` flag you can move all logs to database at `destination_db_url` URL (which is required in this case). When you run gem in this mode no files are created and no other tables are being touched.
+
+Using `--remove_orphans` flag you can remove all orphaned data from tables. When you run gem in this mode no files are created.
+
+Using `--dry_run` flag you can check which data would be removed by gem, but without removing them actually. Instead of that reports will be printed on standard output. This flag can be also combined with `--move_logs` or `--remove_orphans`.
+
+### Configuration options
 
-One of the ways you can configure your export is a file `config/settinigs.yml` that you should place in your app's main directory. The gem uses the properties in following format:
+Despite of command line arguments, one of the ways you can configure your export is a file `config/settings.yml` that you can place in your app's main directory. The gem expects properties in the following format:
 
 ```
 backup:
@@ -51,15 +75,19 @@ backup:
   limit: 1000               # builds limit for one backup file
   threshold: 6              # number of months from now - data younger than this time won't be backuped
   files_location: './dump'  # path of the folder in which backup files will be placed
-  user_id                   # run only for given user
-  org_id                    # run only for given organization
-  repo_id                   # run only for given repository
+  user_id: 1                # run only for given user
+  org_id: 1                 # run only for given organization
+  repo_id: 1                # run only for given repository
+  move_logs: false          # run in move logs mode - move all logs to database at destination_db_url URL
+  remove_orphans: false     # run in remove orphans mode
 ```
 
-You can also set these properties as hash arguments while creating `Backup` instance or use env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`.
+You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`.
 
 You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema.
 
+For `move_logs` mode you need also to specify a destination database. You can set it also in `config/database.yml` file, in `destination` subsection, setting the `destination_db_url` hash argument while creating `Backup` instance or using the `BACKUP_DESTINATION_DB_URL` env var. Your destination database should be consistent with the Travis 3.0 database schema.
+
 ### How to run the test suite
 
 You can run the test after cloning this repository. Next you should call
@@ -74,35 +102,9 @@ and
 bundle exec rspec
 ```
 
-To make tests working properly you should also ensure the database connection string for an empty test database. You can set it as `DATABASE_URL` environment variable or in `config/database.yml`.
-
-**Warning: this database will be cleaned during tests, so ensure that it includes no important data.**
-
-#### Using as standalone application
+To make tests working properly you should also ensure database connection strings for empty test databases. You can set them as `DATABASE_URL` and `BACKUP_DESTINATION_DB_URL` environment variables or in `config/database.yml`.
 
-After cloning this repo you can also run it as a standalone app using
-
-```
-bundle exec bin/travis_backup
-```
-
-You can also pass arguments:
-
-```
-  first argument, no flag    # database url
-  -b, --backup               # when not present, removes data without saving it to file
-  -d, --dry_run              # only prints in console what data will be backuped and deleted
-  -l, --limit LIMIT          # builds limit for one backup file
-  -t, --threshold MONTHS     # number of months from now - data younger than this time won't be backuped
-  -f, --files_location PATH  # path of the folder in which backup files will be placed
-  -u, --user_id ID           # run only for given user
-  -o, --org_id ID            # run only for given organization
-  -r, --repo_id ID           # run only for given repository
-
-  # example:
-
-  bundle exec bin/travis_backup 'postgres://user:pass@localhost:5432/my_db' -b
-```
+**Warning: these databases will be cleaned during tests, so ensure that they include no important data.**
 
 ### Ruby version
 

data/config/database.yml

@@ -18,3 +18,6 @@ test:
   <<: *default
   url: 'postgresql://localhost/travis_test'
   eager_load: false
+  destination:
+    url: 'postgresql://localhost/travis_test_destination'
+    eager_load: false

data/lib/config.rb

@@ -2,7 +2,18 @@
 require 'optparse'
 
 class Config
-  attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id
+  attr_reader :if_backup,
+    :dry_run,
+    :limit,
+    :threshold,
+    :files_location,
+    :database_url,
+    :user_id,
+    :repo_id,
+    :org_id,
+    :move_logs,
+    :remove_orphans,
+    :destination_db_url
 
   def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
     set_values(args)
@@ -43,7 +54,7 @@ class Config
     @files_location = first_not_nil(
       args[:files_location],
       argv_opts[:files_location],
-      ENV['BACKUP_BACKUP_FILES_LOCATION'],
+      ENV['BACKUP_FILES_LOCATION'],
       config.dig('backup', 'files_location'),
       './dump'
     )
@@ -56,25 +67,45 @@ class Config
     @user_id = first_not_nil(
       args[:user_id],
       argv_opts[:user_id],
-      ENV['USER_ID'],
+      ENV['BACKUP_USER_ID'],
       config.dig('backup', 'user_id')
     )
     @repo_id = first_not_nil(
       args[:repo_id],
       argv_opts[:repo_id],
-      ENV['REPO_ID'],
+      ENV['BACKUP_REPO_ID'],
       config.dig('backup', 'repo_id')
     )
     @org_id = first_not_nil(
       args[:org_id],
       argv_opts[:org_id],
-      ENV['ORG_ID'],
+      ENV['BACKUP_ORG_ID'],
       config.dig('backup', 'org_id')
     )
+    @move_logs = first_not_nil(
+      args[:move_logs],
+      argv_opts[:move_logs],
+      ENV['BACKUP_MOVE_LOGS'],
+      config.dig('backup', 'move_logs'),
+      false
+    )
+    @remove_orphans = first_not_nil(
+      args[:remove_orphans],
+      argv_opts[:remove_orphans],
+      ENV['BACKUP_REMOVE_ORPHANS'],
+      config.dig('backup', 'remove_orphans'),
+      false
+    )
+    @destination_db_url = first_not_nil(
+      args[:destination_db_url],
+      argv_opts[:destination_db_url],
+      ENV['BACKUP_DESTINATION_DB_URL'],
+      connection_details.dig(ENV['RAILS_ENV'], 'destination')
+    )
   end
 
   def check_values
-    if !@threshold
+    if !@move_logs && !@remove_orphans && !@threshold
       message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
         "Threshold defines number of months from now.")
       abort message
@@ -84,12 +115,22 @@ class Config
       message = abort_message("Please provide proper database URL.")
       abort message
     end
+
+    if (@move_logs && !@destination_db_url)
+      abort "\nFor moving logs you need to specify your destination database. Example usage:\n" +
+      "\n  $ bin/travis_backup 'postgres://source_url' --move_logs --destination_db_url 'postgres://destination_url'\n" +
+      "\nor using in code:\n" +
+      "\n  Backup.new(database_url: 'postgres://source_url', destination_db_url: 'postgres://destination_url', move_logs: true)\n" +
+      "\nYou can also set it using environment variables or configuration files.\n"  
+    end
   end
 
   def abort_message(intro)
     "\n#{intro} Example usage:\n"+
     "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
-    "\nYou can also set it using environment variables or config/database.yml file.\n"
+    "\nor using in code:\n" +
+    "\n  Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" +
+    "\nYou can also set it using environment variables or configuration files.\n"
   end
 
   def argv_options
@@ -104,6 +145,9 @@ class Config
       opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i }
       opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
       opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
+      opt.on('--move_logs') { |o| options[:move_logs] = o }
+      opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
+      opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
     end.parse!
 
     options[:database_url] = ARGV.shift if ARGV[0]

data/lib/models/branch.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Branch < Model
+  self.table_name = 'branches'
+end

data/lib/models/build.rb

@@ -7,7 +7,7 @@ require 'models/repository'
 # Build model
 class Build < Model
   belongs_to :repository
-  has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :delete_all, class_name: 'Job'
+  has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job'
 
   self.table_name = 'builds'
 end

data/lib/models/commit.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Commit < Model
+  self.table_name = 'commits'
+end

data/lib/models/cron.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Cron < Model
+  self.table_name = 'crons'
+end

data/lib/models/job.rb

@@ -2,12 +2,14 @@
 
 require 'models/model'
 require 'models/repository'
+require 'models/log'
 
 # Job model
 class Job < Model
   self.inheritance_column = :_type_disabled
 
   belongs_to :repository
+  has_many   :logs, -> { order('id') }, foreign_key: :job_id, dependent: :destroy, class_name: 'Log'
 
   self.table_name = 'jobs'
 end

data/lib/models/log.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Log < Model
+  belongs_to :job
+
+  self.table_name = 'logs'
+end

data/lib/models/pull_request.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class PullRequest < Model
+  self.table_name = 'pull_requests'
+end

data/lib/models/repository.rb

@@ -2,10 +2,12 @@
 
 require 'models/model'
 require 'models/build'
+require 'models/request'
 
 # Repository model
 class Repository < Model
   has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build'
+  has_many :requests, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Request'
 
   self.table_name = 'repositories'
 end

data/lib/models/request.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'models/model'
+require 'models/repository'
+
+class Request < Model
+  belongs_to :repository
+
+  self.table_name = 'requests'
+end

data/lib/models/ssl_key.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class SslKey < Model
+  self.table_name = 'ssl_keys'
+end

data/lib/models/stage.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Stage < Model
+  self.table_name = 'stages'
+end

data/lib/models/tag.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Tag < Model
+  self.table_name = 'tags'
+end

data/lib/travis-backup.rb

@@ -4,21 +4,33 @@ require 'active_support/core_ext/array'
 require 'active_support/time'
 require 'config'
 require 'models/repository'
+require 'models/log'
+require 'models/branch'
+require 'models/tag'
+require 'models/commit'
+require 'models/cron'
+require 'models/pull_request'
+require 'models/ssl_key'
+require 'models/request'
+require 'models/stage'
 
 # main travis-backup class
 class Backup
+  attr_accessor :config
+  attr_reader :dry_run_report
+
   def initialize(config_args={})
     @config = Config.new(config_args)
 
     if @config.dry_run
-      @dry_run_removed = {builds: [], jobs: []}
+      @dry_run_report = {builds: [], jobs: [], logs: [], requests: []}
     end
 
     connect_db
   end
 
-  def connect_db
-    ActiveRecord::Base.establish_connection(@config.database_url)
+  def connect_db(url=@config.database_url)
+    ActiveRecord::Base.establish_connection(url)
   end
 
   def run(args={})
@@ -32,63 +44,204 @@ class Backup
     elsif org_id
       owner_id = org_id
       owner_type = 'Organization'
-    elsif repo_id
-      repo_id = repo_id
     end
 
-    if owner_id
-      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-        process_repo(repository)
-      end
+    if @config.move_logs
+      move_logs
+    elsif @config.remove_orphans
+      remove_orphans
+    elsif owner_id
+      process_repos_for_owner(owner_id, owner_type)
     elsif repo_id
-      repository = Repository.find(repo_id)
+      process_repo_with_id(repo_id)
+    else
+      process_all_repos
+    end
+
+    print_dry_run_report if @config.dry_run
+  end
+
+  def process_repos_for_owner(owner_id, owner_type)
+    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
       process_repo(repository)
+    end
+  end
+
+  def process_repo_with_id(repo_id)
+    process_repo(Repository.find(repo_id))
+  end
+
+  def process_all_repos
+    Repository.order(:id).each do |repository|
+      process_repo(repository)
+    end
+  end
+
+  def print_dry_run_report
+    if @dry_run_report.to_a.map(&:second).flatten.empty?
+      puts 'Dry run active. No data would be removed in normal run.'
     else
-      Repository.order(:id).each do |repository|
-        process_repo(repository)
+      puts 'Dry run active. The following data would be removed in normal run:'
+
+      @dry_run_report.to_a.map(&:first).each do |symbol|
+        print_dry_run_report_line(symbol)
       end
     end
+  end
 
-    if @config.dry_run
-      puts 'Dry run active. The following data would be removed in normal mode:'
-      puts " - builds: #{@dry_run_removed[:builds].to_json}"
-      puts " - jobs: #{@dry_run_removed[:jobs].to_json}"
-    end
+  def print_dry_run_report_line(symbol)
+    puts " - #{symbol}: #{@dry_run_report[symbol].to_json}" if @dry_run_report[symbol].any?
+  end
+
+  def process_repo(repository)
+    process_repo_builds(repository)
+    process_repo_requests(repository)
   end
 
-  def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     threshold = @config.threshold.to_i.months.ago.to_datetime
     current_build_id = repository.current_build_id || -1
     repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
               .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch)
+      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
+    end.compact
+  end
+
+  def process_repo_requests(repository)
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    repository.requests.where('created_at < ?', threshold)
+              .in_groups_of(@config.limit.to_i, false).map do |requests_batch|
+      @config.if_backup ? save_and_destroy_requests_batch(requests_batch, repository) : destroy_requests_batch(requests_batch)
     end.compact
   end
 
+  def move_logs
+    return move_logs_dry if config.dry_run
+
+    connect_db(@config.database_url)
+    Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
+      log_hashes = logs_batch.as_json
+      connect_db(@config.destination_db_url)
+
+      log_hashes.each do |log_hash|
+        new_log = Log.new(log_hash)
+        new_log.save!
+      end
+
+      connect_db(@config.database_url)
+
+      logs_batch.each(&:destroy)
+    end
+  end
+
+  def move_logs_dry
+    dry_run_report[:logs].concat(Log.order(:id).map(&:id))
+  end
+
+  def remove_orphans
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'current_build_id')
+    remove_orphans_for_table(Repository, 'repositories', 'builds', 'last_build_id')
+    remove_orphans_for_table(Build, 'builds', 'repositories', 'repository_id')
+    remove_orphans_for_table(Build, 'builds', 'commits', 'commit_id')
+    remove_orphans_for_table(Build, 'builds', 'requests', 'request_id')
+    remove_orphans_for_table(Build, 'builds', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Build, 'builds', 'branches', 'branch_id')
+    remove_orphans_for_table(Build, 'builds', 'tags', 'tag_id')
+    remove_orphans_for_table(Job, 'jobs', 'repositories', 'repository_id')
+    remove_orphans_for_table(Job, 'jobs', 'commits', 'commit_id')
+    remove_orphans_for_table(Job, 'jobs', 'stages', 'stage_id')
+    remove_orphans_for_table(Branch, 'branches', 'repositories', 'repository_id')
+    remove_orphans_for_table(Branch, 'branches', 'builds', 'last_build_id')
+    remove_orphans_for_table(Tag, 'tags', 'repositories', 'repository_id')
+    remove_orphans_for_table(Tag, 'tags', 'builds', 'last_build_id')
+    remove_orphans_for_table(Commit, 'commits', 'repositories', 'repository_id')
+    remove_orphans_for_table(Commit, 'commits', 'branches', 'branch_id')
+    remove_orphans_for_table(Commit, 'commits', 'tags', 'tag_id')
+    remove_orphans_for_table(Cron, 'crons', 'branches', 'branch_id')
+    remove_orphans_for_table(PullRequest, 'pull_requests', 'repositories', 'repository_id')
+    remove_orphans_for_table(SslKey, 'ssl_keys', 'repositories', 'repository_id')
+    remove_orphans_for_table(Request, 'requests', 'commits', 'commit_id')
+    remove_orphans_for_table(Request, 'requests', 'pull_requests', 'pull_request_id')
+    remove_orphans_for_table(Request, 'requests', 'branches', 'branch_id')
+    remove_orphans_for_table(Request, 'requests', 'tags', 'tag_id')
+    remove_orphans_for_table(Stage, 'stages', 'builds', 'build_id')
+  end
+
+  def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
+    for_delete = model_class.find_by_sql(%{
+      select a.*
+      from #{table_a_name} a
+      left join #{table_b_name} b
+      on a.#{fk_name} = b.id
+      where
+        a.#{fk_name} is not null
+        and b.id is null;
+    })
+
+    if config.dry_run
+      key = table_a_name.to_sym
+      dry_run_report[key] = [] if dry_run_report[key].nil?
+      dry_run_report[key].concat(for_delete.map(&:id))
+      dry_run_report[key].uniq!
+    else
+      model_class.where(id: for_delete.map(&:id)).delete_all
+    end
+  end
+
   private
 
-  def save_and_destroy_batch(builds_batch, repository)
+  def save_and_destroy_builds_batch(builds_batch, repository)
     builds_export = export_builds(builds_batch)
     file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
     pretty_json = JSON.pretty_generate(builds_export)
     if save_file(file_name, pretty_json)
-      destroy_batch(builds_batch)
+      destroy_builds_batch(builds_batch)
     end
     builds_export
   end
 
-  def destroy_batch(builds_batch)
-    return destroy_batch_dry(builds_batch) if @config.dry_run
+  def destroy_builds_batch(builds_batch)
+    return destroy_builds_batch_dry(builds_batch) if @config.dry_run
 
     builds_batch.each(&:destroy)
   end
 
-  def destroy_batch_dry(builds_batch)
-    @dry_run_removed[:builds].concat(builds_batch.map(&:id))
+  def destroy_builds_batch_dry(builds_batch)
+    @dry_run_report[:builds].concat(builds_batch.map(&:id))
+
     jobs = builds_batch.map do |build|
       build.jobs.map(&:id) || []
     end.flatten
-    @dry_run_removed[:jobs].concat(jobs)
+
+    @dry_run_report[:jobs].concat(jobs)
+
+    logs = builds_batch.map do |build|
+      build.jobs.map do |job|
+        job.logs.map(&:id) || []
+      end.flatten || []
+    end.flatten
+
+    @dry_run_report[:logs].concat(logs)
+  end
+
+  def save_and_destroy_requests_batch(requests_batch, repository)
+    requests_export = export_requests(requests_batch)
+    file_name = "repository_#{repository.id}_requests_#{requests_batch.first.id}-#{requests_batch.last.id}.json"
+    pretty_json = JSON.pretty_generate(requests_export)
+    if save_file(file_name, pretty_json)
+      destroy_requests_batch(requests_batch)
+    end
+    requests_export
+  end
+
+  def destroy_requests_batch(requests_batch)
+    return destroy_requests_batch_dry(requests_batch) if @config.dry_run
+
+    requests_batch.each(&:destroy)
+  end
+
+  def destroy_requests_batch_dry(requests_batch)
+    @dry_run_report[:requests].concat(requests_batch.map(&:id))
   end
 
   def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
@@ -127,8 +280,21 @@ class Backup
   def export_jobs(jobs)
     jobs.map do |job|
       job_export = job.attributes
+      job_export[:logs] = export_logs(job.logs)
 
       job_export
     end
   end
+
+  def export_logs(logs)
+    logs.map do |log|
+      log.attributes
+    end
+  end
+
+  def export_requests(requests)
+    requests.map do |request|
+      request.attributes
+    end
+  end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.0.1'
+  s.version               = '0.1.1'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-24 00:00:00.000000000 Z
+date: 2021-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -248,11 +248,20 @@ files:
 - db/schema.sql
 - dump/.keep
 - lib/config.rb
+- lib/models/branch.rb
 - lib/models/build.rb
+- lib/models/commit.rb
+- lib/models/cron.rb
 - lib/models/job.rb
+- lib/models/log.rb
 - lib/models/model.rb
 - lib/models/organization.rb
+- lib/models/pull_request.rb
 - lib/models/repository.rb
+- lib/models/request.rb
+- lib/models/ssl_key.rb
+- lib/models/stage.rb
+- lib/models/tag.rb
 - lib/models/user.rb
 - lib/travis-backup.rb
 - log/.keep