travis-backup 0.0.1

data/lib/config.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+require 'optparse'
+
+class Config
+  attr_reader :if_backup, :dry_run, :limit, :threshold, :files_location, :database_url, :user_id, :repo_id, :org_id
+
+  def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+    set_values(args)
+    check_values
+  end
+
+  def set_values(args)
+    config = yaml_load('config/settings.yml')
+    connection_details = yaml_load('config/database.yml')
+    argv_opts = argv_options
+    @if_backup = first_not_nil(
+      args[:if_backup],
+      argv_opts[:if_backup],
+      ENV['IF_BACKUP'],
+      config.dig('backup', 'if_backup'),
+      true
+    )
+    @dry_run = first_not_nil(
+      args[:dry_run],
+      argv_opts[:dry_run],
+      ENV['BACKUP_DRY_RUN'],
+      config.dig('backup', 'dry_run'),
+      false
+    )
+    @limit = first_not_nil(
+      args[:limit],
+      argv_opts[:limit],
+      ENV['BACKUP_LIMIT'],
+      config.dig('backup', 'limit'),
+      1000
+    )
+    @threshold = first_not_nil(
+      args[:threshold],
+      argv_opts[:threshold],
+      ENV['BACKUP_THRESHOLD'],
+      config.dig('backup', 'threshold')
+    )
+    @files_location = first_not_nil(
+      args[:files_location],
+      argv_opts[:files_location],
+      ENV['BACKUP_BACKUP_FILES_LOCATION'],
+      config.dig('backup', 'files_location'),
+      './dump'
+    )
+    @database_url = first_not_nil(
+      args[:database_url],
+      argv_opts[:database_url],
+      ENV['DATABASE_URL'],
+      connection_details.dig(ENV['RAILS_ENV'])
+    )
+    @user_id = first_not_nil(
+      args[:user_id],
+      argv_opts[:user_id],
+      ENV['USER_ID'],
+      config.dig('backup', 'user_id')
+    )
+    @repo_id = first_not_nil(
+      args[:repo_id],
+      argv_opts[:repo_id],
+      ENV['REPO_ID'],
+      config.dig('backup', 'repo_id')
+    )
+    @org_id = first_not_nil(
+      args[:org_id],
+      argv_opts[:org_id],
+      ENV['ORG_ID'],
+      config.dig('backup', 'org_id')
+    )
+  end
+
+  def check_values
+    if !@threshold
+      message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
+        "Threshold defines number of months from now.")
+      abort message
+    end
+
+    if !@database_url
+      message = abort_message("Please provide proper database URL.")
+      abort message
+    end
+  end
+
+  def abort_message(intro)
+    "\n#{intro} Example usage:\n"+
+    "\n  $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
+    "\nYou can also set it using environment variables or config/database.yml file.\n"
+  end
+
+  def argv_options
+    argv_copy = ARGV.clone
+    options = {}
+    OptionParser.new do |opt|
+      opt.on('-b', '--backup') { |o| options[:if_backup] = o }
+      opt.on('-d', '--dry_run') { |o| options[:dry_run] = o }
+      opt.on('-l', '--limit X') { |o| options[:limit] = o.to_i }
+      opt.on('-t', '--threshold X') { |o| options[:threshold] = o.to_i }
+      opt.on('-f', '--files_location X') { |o| options[:files_location] = o }
+      opt.on('-u', '--user_id X') { |o| options[:user_id] = o.to_i }
+      opt.on('-r', '--repo_id X') { |o| options[:repo_id] = o.to_i }
+      opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
+    end.parse!
+
+    options[:database_url] = ARGV.shift if ARGV[0]
+    argv_copy.each do |arg|
+      ARGV.push(arg)
+    end
+    options
+  end
+
+  def first_not_nil(*arr)
+    arr.compact.first
+  end
+
+  def yaml_load(url)
+    begin
+      YAML.load(File.open(url))
+    rescue => e
+      {}
+    end
+  end
+end

data/lib/models/build.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'models/job'
+require 'models/model'
+require 'models/repository'
+
+# Build model
+class Build < Model
+  belongs_to :repository
+  has_many   :jobs, -> { order('id') }, foreign_key: :source_id, dependent: :delete_all, class_name: 'Job'
+
+  self.table_name = 'builds'
+end

data/lib/models/job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'models/model'
+require 'models/repository'
+
+# Job model
+class Job < Model
+  self.inheritance_column = :_type_disabled
+
+  belongs_to :repository
+
+  self.table_name = 'jobs'
+end

data/lib/models/model.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'active_record'
+
+# Model class
+class Model < ActiveRecord::Base
+  self.abstract_class = true
+end

data/lib/models/organization.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class Organization < Model
+  self.table_name = 'organizations'
+end

data/lib/models/repository.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'models/model'
+require 'models/build'
+
+# Repository model
+class Repository < Model
+  has_many :builds, -> { order('id') }, foreign_key: :repository_id, dependent: :destroy, class_name: 'Build'
+
+  self.table_name = 'repositories'
+end

data/lib/models/user.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'models/model'
+
+class User < Model
+  self.table_name = 'users'
+end

data/lib/travis-backup.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/array'
+require 'active_support/time'
+require 'config'
+require 'models/repository'
+
+# main travis-backup class
+class Backup
+  def initialize(config_args={})
+    @config = Config.new(config_args)
+
+    if @config.dry_run
+      @dry_run_removed = {builds: [], jobs: []}
+    end
+
+    connect_db
+  end
+
+  def connect_db
+    ActiveRecord::Base.establish_connection(@config.database_url)
+  end
+
+  def run(args={})
+    user_id = args[:user_id] || @config.user_id
+    repo_id = args[:repo_id] || @config.repo_id
+    org_id = args[:org_id] || @config.org_id
+
+    if user_id
+      owner_id = user_id
+      owner_type = 'User'
+    elsif org_id
+      owner_id = org_id
+      owner_type = 'Organization'
+    elsif repo_id
+      repo_id = repo_id
+    end
+
+    if owner_id
+      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
+        process_repo(repository)
+      end
+    elsif repo_id
+      repository = Repository.find(repo_id)
+      process_repo(repository)
+    else
+      Repository.order(:id).each do |repository|
+        process_repo(repository)
+      end
+    end
+
+    if @config.dry_run
+      puts 'Dry run active. The following data would be removed in normal mode:'
+      puts " - builds: #{@dry_run_removed[:builds].to_json}"
+      puts " - jobs: #{@dry_run_removed[:jobs].to_json}"
+    end
+  end
+
+  def process_repo(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    threshold = @config.threshold.to_i.months.ago.to_datetime
+    current_build_id = repository.current_build_id || -1
+    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
+              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
+      @config.if_backup ? save_and_destroy_batch(builds_batch, repository) : destroy_batch(builds_batch)
+    end.compact
+  end
+
+  private
+
+  def save_and_destroy_batch(builds_batch, repository)
+    builds_export = export_builds(builds_batch)
+    file_name = "repository_#{repository.id}_builds_#{builds_batch.first.id}-#{builds_batch.last.id}.json"
+    pretty_json = JSON.pretty_generate(builds_export)
+    if save_file(file_name, pretty_json)
+      destroy_batch(builds_batch)
+    end
+    builds_export
+  end
+
+  def destroy_batch(builds_batch)
+    return destroy_batch_dry(builds_batch) if @config.dry_run
+
+    builds_batch.each(&:destroy)
+  end
+
+  def destroy_batch_dry(builds_batch)
+    @dry_run_removed[:builds].concat(builds_batch.map(&:id))
+    jobs = builds_batch.map do |build|
+      build.jobs.map(&:id) || []
+    end.flatten
+    @dry_run_removed[:jobs].concat(jobs)
+  end
+
+  def save_file(file_name, content) # rubocop:disable Metrics/MethodLength
+    return true if @config.dry_run
+
+    saved = false
+    begin
+      unless File.directory?(@config.files_location)
+        FileUtils.mkdir_p(@config.files_location)
+      end
+
+      File.open(file_path(file_name), 'w') do |file|
+        file.write(content)
+        file.close
+        saved = true
+      end
+    rescue => e
+      print "Failed to save #{file_name}, error: #{e.inspect}\n"
+    end
+    saved
+  end
+
+  def file_path(file_name)
+    "#{@config.files_location}/#{file_name}"
+  end
+
+  def export_builds(builds)
+    builds.map do |build|
+      build_export = build.attributes
+      build_export[:jobs] = export_jobs(build.jobs)
+
+      build_export
+    end
+  end
+
+  def export_jobs(jobs)
+    jobs.map do |job|
+      job_export = job.attributes
+
+      job_export
+    end
+  end
+end

data/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "travis-backup",
+  "private": true,
+  "dependencies": {
+    "@rails/ujs": "^6.0.0",
+    "turbolinks": "^5.2.0",
+    "@rails/activestorage": "^6.0.0",
+    "@rails/actioncable": "^6.0.0"
+  },
+  "version": "0.1.0"
+}

data/travis-backup.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name                  = 'travis-backup'
+  s.version               = '0.0.1'
+  s.summary               = 'Travis CI backup tool'
+  s.authors               = ['Karol Selak']
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  s.files                 = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  s.executables           = Dir.glob('bin/*').map { |f| File.basename(f) }
+  s.require_paths         = ["lib"]
+  s.license               = 'Beerware'
+
+  s.add_dependency 'activerecord'
+  s.add_dependency 'pg'
+  s.add_dependency 'pry'
+  s.add_dependency 'rails'
+
+  s.add_dependency 'bootsnap'
+  s.add_dependency 'tzinfo-data'
+
+  s.add_development_dependency 'brakeman'
+  s.add_development_dependency 'byebug'
+  s.add_development_dependency 'factory_bot'
+  s.add_development_dependency 'rspec-rails'
+  s.add_development_dependency 'listen'
+  s.add_development_dependency 'rubocop', '~> 0.75.1'
+  s.add_development_dependency 'rubocop-rspec'
+end

metadata

@@ -0,0 +1,286 @@
+--- !ruby/object:Gem::Specification
+name: travis-backup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Karol Selak
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bootsnap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tzinfo-data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: brakeman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.75.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.75.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables:
+- setup
+- rails
+- spring
+- yarn
+- bundle
+- travis_backup
+- rake
+- console
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/bundle
+- bin/console
+- bin/rails
+- bin/rake
+- bin/setup
+- bin/spring
+- bin/travis_backup
+- bin/yarn
+- config.ru
+- config/application.rb
+- config/boot.rb
+- config/cable.yml
+- config/credentials.yml.enc
+- config/database.yml
+- config/environment.rb
+- config/environments/development.rb
+- config/environments/production.rb
+- config/environments/test.rb
+- config/initializers/application_controller_renderer.rb
+- config/initializers/assets.rb
+- config/initializers/backtrace_silencers.rb
+- config/initializers/content_security_policy.rb
+- config/initializers/cookies_serializer.rb
+- config/initializers/filter_parameter_logging.rb
+- config/initializers/inflections.rb
+- config/initializers/mime_types.rb
+- config/initializers/permissions_policy.rb
+- config/initializers/wrap_parameters.rb
+- config/locales/en.yml
+- config/puma.rb
+- config/routes.rb
+- config/settings.yml
+- config/spring.rb
+- db/schema.sql
+- dump/.keep
+- lib/config.rb
+- lib/models/build.rb
+- lib/models/job.rb
+- lib/models/model.rb
+- lib/models/organization.rb
+- lib/models/repository.rb
+- lib/models/user.rb
+- lib/travis-backup.rb
+- log/.keep
+- package.json
+- tmp/.keep
+- travis-backup.gemspec
+- vendor/.keep
+homepage: 
+licenses:
+- Beerware
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Travis CI backup tool
+test_files: []