travis-backup 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82c4a389c3ed4d73db6f9b50b4d80a17524f646b1ce4fa6737821a8e4cc7009d
-  data.tar.gz: 514f7b4042b0ef3297371ec5062f3d0ebcb001b1916f35b2c66e0ea6ffb4a201
+  metadata.gz: dca7a53c9e3689c5f0988d074296449769a106b86ed6ddc26c6a8b7c0cba15b2
+  data.tar.gz: 25765ff21af6e3bbba2fbf5a971d678acd6447b59c54fc685ddaa0912e84f73a
 SHA512:
-  metadata.gz: fc3c44b552f31d73d62469cb619c516297ed7a3b02ea4694c8bc39cd2fd48cecb80a5919ec5299f0ec2ea2fc4b31c7bfcc0182cce22721d685b68f1fae921c20
-  data.tar.gz: 3440dff96a5940425568de23c2c714f663e2b351694d6dcd8ad70646f8627e69258386b2e415da3172cdde5ddb5b126019d555722de541d183edc187ee42289c
+  metadata.gz: bcc37fc6219ac84eda44f953879952567eb33981a9a2436b5922acefa515994262ab174e25e8fe9660153e73e3d55838eaa8fd375dc8f79cefff02f574f8e516
+  data.tar.gz: 377cef9823245bd5153bc291e6fb078c723544986e6fd85a79e50b64d8cc5110c1331ac8247a40b158022633672ad93e3a85719d2b70f4659e152913f0b4b729

data/.travis.yml

@@ -30,4 +30,7 @@ before_script:
   - psql --version
   - psql -c 'CREATE DATABASE travis_test;' -U postgres
   - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
-  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test
+  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test
+  - psql -c 'CREATE DATABASE travis_test_destination;' -U postgres
+  - psql -t -c "SELECT 1 FROM pg_roles WHERE rolname='travis'" -U postgres | grep 1 || psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
+  - psql -f db/schema.sql -v ON_ERROR_STOP=1 travis_test_destination

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    travis-backup (0.0.1)
+    travis-backup (0.0.3)
       activerecord
       bootsnap
       pg

data/config/database.yml

@@ -18,3 +18,6 @@ test:
   <<: *default
   url: 'postgresql://localhost/travis_test'
   eager_load: false
+  destination:
+    url: 'postgresql://localhost/travis_test_destination'
+    eager_load: false

data/lib/config.rb

@@ -2,7 +2,17 @@
 require 'optparse'
 
 class Config
-  attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id
+  attr_reader :if_backup,
+    :dry_run,
+    :limit,
+    :threshold,
+    :files_location,
+    :database_url,
+    :user_id,
+    :repo_id,
+    :org_id,
+    :move_logs,
+    :destination_db_url
 
   def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
     set_values(args)
@@ -43,7 +53,7 @@ class Config
     @files_location = first_not_nil(
       args[:files_location],
       argv_opts[:files_location],
-      ENV['BACKUP_BACKUP_FILES_LOCATION'],
+      ENV['BACKUP_FILES_LOCATION'],
       config.dig('backup', 'files_location'),
       './dump'
     )
@@ -71,10 +81,23 @@ class Config
       ENV['ORG_ID'],
       config.dig('backup', 'org_id')
     )
+    @move_logs = first_not_nil(
+      args[:move_logs],
+      argv_opts[:move_logs],
+      ENV['BACKUP_MOVE_LOGS'],
+      config.dig('backup', 'move_logs'),
+      false
+    )
+    @destination_db_url = first_not_nil(
+      args[:destination_db_url],
+      argv_opts[:destination_db_url],
+      ENV['BACKUP_DESTINATION_DB_URL'],
+      connection_details.dig(ENV['RAILS_ENV'], 'destination')
+    )
   end
 
   def check_values
-    if !@threshold
+    if !@move_logs && !@threshold
       message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
         "Threshold defines number of months from now.")
       abort message
@@ -84,6 +107,14 @@ class Config
       message = abort_message("Please provide proper database URL.")
       abort message
     end
+
+    if (@move_logs && !@destination_db_url)
+      abort "\nFor moving logs you need to specify your destination database. Example usage:\n" +
+      "\n  $ bin/travis_backup 'postgres://source_url' --move_logs --destination_db_url 'postgres://destination_url'\n" +
+      "\nor using in code:\n" +
+      "\n  Backup.new(database_url: 'postgres://source_url', destination_db_url: 'postgres://destination_url', move_logs: true)\n" +
+      "\nYou can also set it using environment variables or configuration files.\n"  
+    end
   end
 
   def abort_message(intro)
@@ -91,7 +122,7 @@ class Config
     "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
     "\nor using in code:\n" +
     "\n  Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" +
-    "\nYou can also set it using environment variables or config/database.yml file.\n"
+    "\nYou can also set it using environment variables or configuration files.\n"
   end
 
   def argv_options
@@ -106,6 +137,8 @@ class Config
       opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i }
       opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
       opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
+      opt.on('--move_logs') { |o| options[:move_logs] = o }
+      opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
     end.parse!
 
     options[:database_url] = ARGV.shift if ARGV[0]

data/lib/models/build.rb

@@ -7,7 +7,7 @@ require 'models/repository'
 # Build model
 class Build < Model
   belongs_to :repository
-  has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :delete_all, class_name: 'Job'
+  has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :destroy, class_name: 'Job'
 
   self.table_name = 'builds'
 end

data/lib/models/job.rb

@@ -2,12 +2,14 @@
 
 require 'models/model'
 require 'models/repository'
+require 'models/log'
 
 # Job model
 class Job < Model
   self.inheritance_column = :_type_disabled
 
   belongs_to :repository
+  has_many   :logs, -> { order('id') }, foreign_key: :job_id, dependent: :destroy, class_name: 'Log'
 
   self.table_name = 'jobs'
 end

data/lib/models/log.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Log < Model
+  belongs_to :job
+
+  self.table_name = 'logs'
+end

data/lib/travis-backup.rb

@@ -4,24 +4,30 @@ require 'active_support/core_ext/array'
 require 'active_support/time'
 require 'config'
 require 'models/repository'
+require 'models/log'
 
 # main travis-backup class
 class Backup
+  attr_accessor :config
+  attr_reader :dry_run_removed
+
   def initialize(config_args={})
     @config = Config.new(config_args)
 
     if @config.dry_run
-      @dry_run_removed = {builds: [], jobs: []}
+      @dry_run_removed = {builds: [], jobs: [], logs: []}
     end
 
     connect_db
   end
 
-  def connect_db
-    ActiveRecord::Base.establish_connection(@config.database_url)
+  def connect_db(url=@config.database_url)
+    ActiveRecord::Base.establish_connection(url)
   end
 
   def run(args={})
+    return move_logs if @config.move_logs
+
     user_id = args[:user_id] || @config.user_id
     repo_id = args[:repo_id] || @config.repo_id
     org_id = args[:org_id] || @config.org_id
@@ -53,6 +59,24 @@ class Backup
       puts 'Dry run active. The following data would be removed in normal mode:'
       puts " - builds: #{@dry_run_removed[:builds].to_json}"
       puts " - jobs: #{@dry_run_removed[:jobs].to_json}"
+      puts " - logs: #{@dry_run_removed[:logs].to_json}"
+    end
+  end
+
+  def move_logs
+    connect_db(@config.database_url)
+    Log.order(:id).in_groups_of(@config.limit.to_i, false).map do |logs_batch|
+      log_hashes = logs_batch.as_json
+      connect_db(@config.destination_db_url)
+
+      log_hashes.each do |log_hash|
+        new_log = Log.new(log_hash)
+        new_log.save!
+      end
+
+      connect_db(@config.database_url)
+
+      logs_batch.each(&:destroy)
     end
   end
 
@@ -85,10 +109,20 @@ class Backup
 
   def destroy_batch_dry(builds_batch)
     @dry_run_removed[:builds].concat(builds_batch.map(&:id))
+
     jobs = builds_batch.map do |build|
       build.jobs.map(&:id) || []
     end.flatten
+
     @dry_run_removed[:jobs].concat(jobs)
+
+    logs = builds_batch.map do |build|
+      build.jobs.map do |job|
+        job.logs.map(&:id) || []
+      end.flatten || []
+    end.flatten
+
+    @dry_run_removed[:logs].concat(logs)
   end
 
   def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
@@ -127,8 +161,17 @@ class Backup
   def export_jobs(jobs)
     jobs.map do |job|
       job_export = job.attributes
+      job_export[:logs] = export_logs(job.logs)
 
       job_export
     end
   end
+
+  def export_logs(logs)
+    logs.map do |log|
+      log_export = log.attributes
+
+      log_export
+    end
+  end
 end

data/travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.0.2'
+  s.version               = '0.0.3'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: travis-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Karol Selak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-24 00:00:00.000000000 Z
+date: 2021-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -250,6 +250,7 @@ files:
 - lib/config.rb
 - lib/models/build.rb
 - lib/models/job.rb
+- lib/models/log.rb
 - lib/models/model.rb
 - lib/models/organization.rb
 - lib/models/repository.rb