travis-backup 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +14 -4
- data/lib/backup/load_from_files.rb +245 -0
- data/lib/backup/remove_orphans.rb +81 -50
- data/lib/backup/remove_specified/remove_heavy_data.rb +99 -0
- data/lib/backup/remove_specified/remove_with_all_dependencies.rb +51 -0
- data/lib/backup/remove_specified/shared.rb +20 -0
- data/lib/backup/remove_specified.rb +68 -0
- data/lib/backup/save_file.rb +43 -0
- data/lib/backup/save_id_hash_to_file.rb +33 -0
- data/lib/backup/save_nullified_rels_to_file.rb +29 -0
- data/lib/config.rb +37 -7
- data/lib/dry_run_reporter.rb +18 -4
- data/lib/id_hash.rb +97 -0
- data/lib/ids_of_all_dependencies.rb +330 -0
- data/lib/model.rb +77 -0
- data/lib/models/abuse.rb +9 -0
- data/lib/models/annotation.rb +8 -0
- data/lib/models/branch.rb +9 -1
- data/lib/models/broadcast.rb +8 -0
- data/lib/models/build.rb +23 -3
- data/lib/models/commit.rb +8 -1
- data/lib/models/cron.rb +2 -1
- data/lib/models/email.rb +8 -0
- data/lib/models/invoice.rb +8 -0
- data/lib/models/job.rb +10 -2
- data/lib/models/log.rb +1 -1
- data/lib/models/membership.rb +9 -0
- data/lib/models/message.rb +8 -0
- data/lib/models/organization.rb +15 -1
- data/lib/models/owner_group.rb +8 -0
- data/lib/models/permission.rb +9 -0
- data/lib/models/pull_request.rb +5 -1
- data/lib/models/queueable_job.rb +8 -0
- data/lib/models/repository.rb +16 -3
- data/lib/models/request.rb +11 -1
- data/lib/models/ssl_key.rb +2 -1
- data/lib/models/stage.rb +4 -1
- data/lib/models/star.rb +9 -0
- data/lib/models/subscription.rb +9 -0
- data/lib/models/tag.rb +7 -1
- data/lib/models/token.rb +8 -0
- data/lib/models/trial.rb +9 -0
- data/lib/models/trial_allowance.rb +9 -0
- data/lib/models/user.rb +33 -1
- data/lib/models/user_beta_feature.rb +8 -0
- data/lib/nullify_dependencies.rb +42 -0
- data/lib/travis-backup.rb +29 -5
- data/travis-backup.gemspec +1 -1
- metadata +35 -9
- data/lib/backup/remove_old.rb +0 -204
- data/lib/models/model.rb +0 -8
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'backup/remove_specified/remove_with_all_dependencies'
|
4
|
+
require 'backup/remove_specified/remove_heavy_data'
|
5
|
+
|
6
|
+
class Backup
|
7
|
+
class RemoveSpecified
|
8
|
+
include RemoveHeavyData
|
9
|
+
include RemoveWithAllDependencies
|
10
|
+
|
11
|
+
attr_reader :config
|
12
|
+
|
13
|
+
def initialize(config, dry_run_reporter=nil)
|
14
|
+
@config = config
|
15
|
+
@dry_run_reporter = dry_run_reporter
|
16
|
+
end
|
17
|
+
|
18
|
+
def dry_run_report
|
19
|
+
@dry_run_reporter.report
|
20
|
+
end
|
21
|
+
|
22
|
+
def run(args={})
|
23
|
+
user_id = args[:user_id] || @config.user_id
|
24
|
+
repo_id = args[:repo_id] || @config.repo_id
|
25
|
+
org_id = args[:org_id] || @config.org_id
|
26
|
+
|
27
|
+
if user_id
|
28
|
+
process_user(user_id)
|
29
|
+
elsif org_id
|
30
|
+
process_organization(org_id)
|
31
|
+
elsif repo_id
|
32
|
+
process_repo_with_id(repo_id)
|
33
|
+
else
|
34
|
+
process_all_repos
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def process_user(user_id)
|
39
|
+
if @config.threshold
|
40
|
+
remove_heavy_data_for_repos_owned_by(user_id, 'User')
|
41
|
+
else
|
42
|
+
remove_user_with_dependencies(user_id)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def process_organization(org_id)
|
47
|
+
if @config.threshold
|
48
|
+
remove_heavy_data_for_repos_owned_by(org_id, 'Organization')
|
49
|
+
else
|
50
|
+
remove_org_with_dependencies(org_id)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def process_repo_with_id(repo_id)
|
55
|
+
if @config.threshold
|
56
|
+
remove_heavy_data_for_repo(Repository.find(repo_id))
|
57
|
+
else
|
58
|
+
remove_repo_with_dependencies(repo_id)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def process_all_repos
|
63
|
+
Repository.order(:id).each do |repository|
|
64
|
+
remove_heavy_data_for_repo(repository)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SaveFile
|
4
|
+
def save_file(file_path, content) # rubocop:disable Metrics/MethodLength
|
5
|
+
return true if @config.dry_run
|
6
|
+
|
7
|
+
saved = false
|
8
|
+
begin
|
9
|
+
ensure_path(file_path)
|
10
|
+
|
11
|
+
File.open(full_file_path(file_path), 'w') do |file|
|
12
|
+
file.write(content)
|
13
|
+
file.close
|
14
|
+
saved = true
|
15
|
+
end
|
16
|
+
rescue => e
|
17
|
+
print "Failed to save #{file_path}, error: #{e.inspect}\n"
|
18
|
+
end
|
19
|
+
saved
|
20
|
+
end
|
21
|
+
|
22
|
+
def current_time_for_subfolder
|
23
|
+
Time.now.to_s.parameterize.underscore
|
24
|
+
end
|
25
|
+
|
26
|
+
def ensure_path(file_path)
|
27
|
+
path = folder_path(file_path)
|
28
|
+
|
29
|
+
unless File.directory?(path)
|
30
|
+
FileUtils.mkdir_p(path)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def full_file_path(file_path)
|
35
|
+
"#{@config.files_location}/#{file_path}"
|
36
|
+
end
|
37
|
+
|
38
|
+
def folder_path(file_path)
|
39
|
+
result = full_file_path(file_path).split('/')
|
40
|
+
result.pop
|
41
|
+
result.join('/')
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'backup/save_file'
|
2
|
+
|
3
|
+
module SaveIdHashToFile
|
4
|
+
include SaveFile
|
5
|
+
|
6
|
+
def save_id_hash_to_file(id_hash)
|
7
|
+
id_hash.each do |name, ids|
|
8
|
+
ids.sort.each_slice(@config.limit.to_i) do |ids_batch|
|
9
|
+
save_ids_batch_to_file(name, ids_batch)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def save_ids_batch_to_file(name, ids_batch)
|
15
|
+
model = Model.get_model(name)
|
16
|
+
|
17
|
+
export = {}
|
18
|
+
export[:table_name] = model.table_name
|
19
|
+
export[:data] = ids_batch.map do |id|
|
20
|
+
get_exported_object(model, id)
|
21
|
+
end
|
22
|
+
|
23
|
+
content = JSON.pretty_generate(export)
|
24
|
+
file_name = "#{name}_#{ids_batch.first}-#{ids_batch.last}.json"
|
25
|
+
file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
|
26
|
+
save_file(file_path, content)
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_exported_object(model, id)
|
30
|
+
object = model.find(id)
|
31
|
+
object.attributes
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'backup/save_file'
|
2
|
+
|
3
|
+
module SaveNullifiedRelsToFile
|
4
|
+
include SaveFile
|
5
|
+
|
6
|
+
def save_nullified_rels_to_file(rels_hash)
|
7
|
+
@file_index = 1
|
8
|
+
|
9
|
+
rels_hash.each do |name, rels|
|
10
|
+
rels&.compact&.each_slice(@config.limit.to_i) do |rels_batch|
|
11
|
+
save_rels_batch_to_file(name, rels_batch)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def save_rels_batch_to_file(name, rels_batch)
|
17
|
+
model = Model.get_model(name)
|
18
|
+
|
19
|
+
export = {}
|
20
|
+
export[:table_name] = model.table_name
|
21
|
+
export[:nullified_relationships] = rels_batch
|
22
|
+
|
23
|
+
content = JSON.pretty_generate(export)
|
24
|
+
file_name = "nullified_relationships/build_#{@file_index}.json"
|
25
|
+
@file_index += 1
|
26
|
+
file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
|
27
|
+
save_file(file_path, content)
|
28
|
+
end
|
29
|
+
end
|
data/lib/config.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
require 'optparse'
|
3
3
|
|
4
4
|
class Config
|
5
|
-
|
5
|
+
attr_accessor :if_backup,
|
6
6
|
:dry_run,
|
7
7
|
:limit,
|
8
8
|
:threshold,
|
@@ -13,7 +13,10 @@ class Config
|
|
13
13
|
:org_id,
|
14
14
|
:move_logs,
|
15
15
|
:remove_orphans,
|
16
|
-
:
|
16
|
+
:orphans_table,
|
17
|
+
:destination_db_url,
|
18
|
+
:load_from_files,
|
19
|
+
:id_gap
|
17
20
|
|
18
21
|
def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
|
19
22
|
set_values(args)
|
@@ -96,18 +99,40 @@ class Config
|
|
96
99
|
config.dig('backup', 'remove_orphans'),
|
97
100
|
false
|
98
101
|
)
|
102
|
+
@orphans_table = first_not_nil(
|
103
|
+
args[:orphans_table],
|
104
|
+
argv_opts[:orphans_table],
|
105
|
+
ENV['BACKUP_ORPHANS_TABLE'],
|
106
|
+
config.dig('backup', 'orphans_table'),
|
107
|
+
false
|
108
|
+
)
|
99
109
|
@destination_db_url = first_not_nil(
|
100
110
|
args[:destination_db_url],
|
101
111
|
argv_opts[:destination_db_url],
|
102
112
|
ENV['BACKUP_DESTINATION_DB_URL'],
|
103
113
|
connection_details.dig(ENV['RAILS_ENV'], 'destination')
|
104
114
|
)
|
115
|
+
@load_from_files = first_not_nil(
|
116
|
+
args[:load_from_files],
|
117
|
+
argv_opts[:load_from_files],
|
118
|
+
ENV['BACKUP_LOAD_FROM_FILES'],
|
119
|
+
config.dig('backup', 'load_from_files'),
|
120
|
+
false
|
121
|
+
)
|
122
|
+
@id_gap = first_not_nil(
|
123
|
+
args[:id_gap],
|
124
|
+
argv_opts[:id_gap],
|
125
|
+
ENV['BACKUP_ID_GAP'],
|
126
|
+
config.dig('backup', 'id_gap'),
|
127
|
+
1000
|
128
|
+
)
|
105
129
|
end
|
106
130
|
|
107
131
|
def check_values
|
108
|
-
if !@move_logs && !@remove_orphans && !@threshold
|
132
|
+
if !@move_logs && !@remove_orphans && !@threshold && !@user_id && !@org_id && !@repo_id && !@load_from_files
|
109
133
|
message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
|
110
|
-
"Threshold defines number of months from now."
|
134
|
+
"Threshold defines number of months from now. Alternatively you can define user_id, org_id or repo_id " +
|
135
|
+
"to remove whole user, organization or repository with all dependencies.")
|
111
136
|
abort message
|
112
137
|
end
|
113
138
|
|
@@ -126,10 +151,12 @@ class Config
|
|
126
151
|
end
|
127
152
|
|
128
153
|
def abort_message(intro)
|
129
|
-
"\n#{intro}
|
130
|
-
"\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6
|
154
|
+
"\n#{intro}\n\nExample usage:\n"+
|
155
|
+
"\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6" +
|
156
|
+
"\n $ bin/travis_backup 'postgres://my_database_url' --user_id 1\n" +
|
131
157
|
"\nor using in code:\n" +
|
132
|
-
"\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)
|
158
|
+
"\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)" +
|
159
|
+
"\n Backup.new(database_url: 'postgres://my_database_url', user_id: 1)\n" +
|
133
160
|
"\nYou can also set it using environment variables or configuration files.\n"
|
134
161
|
end
|
135
162
|
|
@@ -147,7 +174,10 @@ class Config
|
|
147
174
|
opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
|
148
175
|
opt.on('--move_logs') { |o| options[:move_logs] = o }
|
149
176
|
opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
|
177
|
+
opt.on('--orphans_table X') { |o| options[:orphans_table] = o }
|
150
178
|
opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
|
179
|
+
opt.on('--load_from_files') { |o| options[:load_from_files] = o }
|
180
|
+
opt.on('--id_gap X') { |o| options[:id_gap] = o.to_i }
|
151
181
|
end.parse!
|
152
182
|
|
153
183
|
options[:database_url] = ARGV.shift if ARGV[0]
|
data/lib/dry_run_reporter.rb
CHANGED
@@ -7,10 +7,12 @@ class DryRunReporter
|
|
7
7
|
@report = {}
|
8
8
|
end
|
9
9
|
|
10
|
-
def add_to_report(
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
def add_to_report(*args)
|
11
|
+
if args.first.is_a?(Hash)
|
12
|
+
add_to_report_as_hash(args.first)
|
13
|
+
else
|
14
|
+
add_to_report_as_key_and_values(*args)
|
15
|
+
end
|
14
16
|
end
|
15
17
|
|
16
18
|
def print_report
|
@@ -27,6 +29,18 @@ class DryRunReporter
|
|
27
29
|
|
28
30
|
private
|
29
31
|
|
32
|
+
def add_to_report_as_key_and_values(key, *values)
|
33
|
+
report[key] = [] if report[key].nil?
|
34
|
+
report[key].concat(values)
|
35
|
+
report[key].uniq!
|
36
|
+
end
|
37
|
+
|
38
|
+
def add_to_report_as_hash(hash)
|
39
|
+
hash.each do |key, array|
|
40
|
+
add_to_report_as_key_and_values(key, *array)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
30
44
|
def print_report_line(symbol)
|
31
45
|
puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any?
|
32
46
|
end
|
data/lib/id_hash.rb
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'model'
|
2
|
+
|
3
|
+
class HashOfArrays < Hash
|
4
|
+
def initialize(hash = {})
|
5
|
+
hash.each do |key, array|
|
6
|
+
self[key] = hash[key].clone
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def clone
|
11
|
+
self.class.new(self)
|
12
|
+
end
|
13
|
+
|
14
|
+
def subtract(hash)
|
15
|
+
result = self.clone
|
16
|
+
hash.each do |key, array|
|
17
|
+
next if result[key].nil?
|
18
|
+
|
19
|
+
array.each do |el|
|
20
|
+
result[key].delete(el)
|
21
|
+
end
|
22
|
+
|
23
|
+
result.delete(key) if result[key].empty?
|
24
|
+
end
|
25
|
+
result
|
26
|
+
end
|
27
|
+
|
28
|
+
def add(key, *values)
|
29
|
+
self[key] = [] if self[key].nil?
|
30
|
+
self[key].concat(values)
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.join(*hashes)
|
34
|
+
result = self.new
|
35
|
+
result.join(*hashes)
|
36
|
+
end
|
37
|
+
|
38
|
+
def join(*hashes)
|
39
|
+
hashes.each do |hash|
|
40
|
+
hash.each do |key, array|
|
41
|
+
self.add(key, *array)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
self
|
45
|
+
end
|
46
|
+
|
47
|
+
def sort_arrays!
|
48
|
+
self.each do |key, array|
|
49
|
+
array.sort!
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
class IdHash < HashOfArrays
|
55
|
+
def add(key, *values)
|
56
|
+
super(key, *values)
|
57
|
+
self[key].uniq!
|
58
|
+
self
|
59
|
+
end
|
60
|
+
|
61
|
+
def remove_entries_from_db(as_first: [], as_last: [])
|
62
|
+
exceptionals = as_first + as_last
|
63
|
+
remove_from_exceptional(as_first)
|
64
|
+
|
65
|
+
self.each do |name, ids|
|
66
|
+
next if exceptionals.include?(name)
|
67
|
+
remove_entries_from_array(name, ids)
|
68
|
+
end
|
69
|
+
|
70
|
+
remove_from_exceptional(as_last)
|
71
|
+
end
|
72
|
+
|
73
|
+
def with_table_symbols
|
74
|
+
result = HashOfArrays.new
|
75
|
+
|
76
|
+
self.each do |name, ids|
|
77
|
+
symbol = Model.get_model(name).table_name.to_sym
|
78
|
+
result[symbol] = ids
|
79
|
+
end
|
80
|
+
|
81
|
+
result
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def remove_from_exceptional(array)
|
87
|
+
array.each do |name|
|
88
|
+
ids = self[name]
|
89
|
+
remove_entries_from_array(name, ids)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def remove_entries_from_array(model_name, ids)
|
94
|
+
model = Model.get_model(model_name)
|
95
|
+
model.delete(ids) if model.present?
|
96
|
+
end
|
97
|
+
end
|