travis-backup 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/README.md +14 -4
  4. data/lib/backup/load_from_files.rb +245 -0
  5. data/lib/backup/remove_orphans.rb +81 -50
  6. data/lib/backup/remove_specified/remove_heavy_data.rb +99 -0
  7. data/lib/backup/remove_specified/remove_with_all_dependencies.rb +51 -0
  8. data/lib/backup/remove_specified/shared.rb +20 -0
  9. data/lib/backup/remove_specified.rb +68 -0
  10. data/lib/backup/save_file.rb +43 -0
  11. data/lib/backup/save_id_hash_to_file.rb +33 -0
  12. data/lib/backup/save_nullified_rels_to_file.rb +29 -0
  13. data/lib/config.rb +37 -7
  14. data/lib/dry_run_reporter.rb +18 -4
  15. data/lib/id_hash.rb +97 -0
  16. data/lib/ids_of_all_dependencies.rb +330 -0
  17. data/lib/model.rb +77 -0
  18. data/lib/models/abuse.rb +9 -0
  19. data/lib/models/annotation.rb +8 -0
  20. data/lib/models/branch.rb +9 -1
  21. data/lib/models/broadcast.rb +8 -0
  22. data/lib/models/build.rb +23 -3
  23. data/lib/models/commit.rb +8 -1
  24. data/lib/models/cron.rb +2 -1
  25. data/lib/models/email.rb +8 -0
  26. data/lib/models/invoice.rb +8 -0
  27. data/lib/models/job.rb +10 -2
  28. data/lib/models/log.rb +1 -1
  29. data/lib/models/membership.rb +9 -0
  30. data/lib/models/message.rb +8 -0
  31. data/lib/models/organization.rb +15 -1
  32. data/lib/models/owner_group.rb +8 -0
  33. data/lib/models/permission.rb +9 -0
  34. data/lib/models/pull_request.rb +5 -1
  35. data/lib/models/queueable_job.rb +8 -0
  36. data/lib/models/repository.rb +16 -3
  37. data/lib/models/request.rb +11 -1
  38. data/lib/models/ssl_key.rb +2 -1
  39. data/lib/models/stage.rb +4 -1
  40. data/lib/models/star.rb +9 -0
  41. data/lib/models/subscription.rb +9 -0
  42. data/lib/models/tag.rb +7 -1
  43. data/lib/models/token.rb +8 -0
  44. data/lib/models/trial.rb +9 -0
  45. data/lib/models/trial_allowance.rb +9 -0
  46. data/lib/models/user.rb +33 -1
  47. data/lib/models/user_beta_feature.rb +8 -0
  48. data/lib/nullify_dependencies.rb +42 -0
  49. data/lib/travis-backup.rb +29 -5
  50. data/travis-backup.gemspec +1 -1
  51. metadata +35 -9
  52. data/lib/backup/remove_old.rb +0 -204
  53. data/lib/models/model.rb +0 -8
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'backup/remove_specified/remove_with_all_dependencies'
4
+ require 'backup/remove_specified/remove_heavy_data'
5
+
6
+ class Backup
7
+ class RemoveSpecified
8
+ include RemoveHeavyData
9
+ include RemoveWithAllDependencies
10
+
11
+ attr_reader :config
12
+
13
+ def initialize(config, dry_run_reporter=nil)
14
+ @config = config
15
+ @dry_run_reporter = dry_run_reporter
16
+ end
17
+
18
+ def dry_run_report
19
+ @dry_run_reporter.report
20
+ end
21
+
22
+ def run(args={})
23
+ user_id = args[:user_id] || @config.user_id
24
+ repo_id = args[:repo_id] || @config.repo_id
25
+ org_id = args[:org_id] || @config.org_id
26
+
27
+ if user_id
28
+ process_user(user_id)
29
+ elsif org_id
30
+ process_organization(org_id)
31
+ elsif repo_id
32
+ process_repo_with_id(repo_id)
33
+ else
34
+ process_all_repos
35
+ end
36
+ end
37
+
38
+ def process_user(user_id)
39
+ if @config.threshold
40
+ remove_heavy_data_for_repos_owned_by(user_id, 'User')
41
+ else
42
+ remove_user_with_dependencies(user_id)
43
+ end
44
+ end
45
+
46
+ def process_organization(org_id)
47
+ if @config.threshold
48
+ remove_heavy_data_for_repos_owned_by(org_id, 'Organization')
49
+ else
50
+ remove_org_with_dependencies(org_id)
51
+ end
52
+ end
53
+
54
+ def process_repo_with_id(repo_id)
55
+ if @config.threshold
56
+ remove_heavy_data_for_repo(Repository.find(repo_id))
57
+ else
58
+ remove_repo_with_dependencies(repo_id)
59
+ end
60
+ end
61
+
62
+ def process_all_repos
63
+ Repository.order(:id).each do |repository|
64
+ remove_heavy_data_for_repo(repository)
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SaveFile
4
+ def save_file(file_path, content) # rubocop:disable Metrics/MethodLength
5
+ return true if @config.dry_run
6
+
7
+ saved = false
8
+ begin
9
+ ensure_path(file_path)
10
+
11
+ File.open(full_file_path(file_path), 'w') do |file|
12
+ file.write(content)
13
+ file.close
14
+ saved = true
15
+ end
16
+ rescue => e
17
+ print "Failed to save #{file_path}, error: #{e.inspect}\n"
18
+ end
19
+ saved
20
+ end
21
+
22
+ def current_time_for_subfolder
23
+ Time.now.to_s.parameterize.underscore
24
+ end
25
+
26
+ def ensure_path(file_path)
27
+ path = folder_path(file_path)
28
+
29
+ unless File.directory?(path)
30
+ FileUtils.mkdir_p(path)
31
+ end
32
+ end
33
+
34
+ def full_file_path(file_path)
35
+ "#{@config.files_location}/#{file_path}"
36
+ end
37
+
38
+ def folder_path(file_path)
39
+ result = full_file_path(file_path).split('/')
40
+ result.pop
41
+ result.join('/')
42
+ end
43
+ end
@@ -0,0 +1,33 @@
1
+ require 'backup/save_file'
2
+
3
+ module SaveIdHashToFile
4
+ include SaveFile
5
+
6
+ def save_id_hash_to_file(id_hash)
7
+ id_hash.each do |name, ids|
8
+ ids.sort.each_slice(@config.limit.to_i) do |ids_batch|
9
+ save_ids_batch_to_file(name, ids_batch)
10
+ end
11
+ end
12
+ end
13
+
14
+ def save_ids_batch_to_file(name, ids_batch)
15
+ model = Model.get_model(name)
16
+
17
+ export = {}
18
+ export[:table_name] = model.table_name
19
+ export[:data] = ids_batch.map do |id|
20
+ get_exported_object(model, id)
21
+ end
22
+
23
+ content = JSON.pretty_generate(export)
24
+ file_name = "#{name}_#{ids_batch.first}-#{ids_batch.last}.json"
25
+ file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
26
+ save_file(file_path, content)
27
+ end
28
+
29
+ def get_exported_object(model, id)
30
+ object = model.find(id)
31
+ object.attributes
32
+ end
33
+ end
@@ -0,0 +1,29 @@
1
+ require 'backup/save_file'
2
+
3
+ module SaveNullifiedRelsToFile
4
+ include SaveFile
5
+
6
+ def save_nullified_rels_to_file(rels_hash)
7
+ @file_index = 1
8
+
9
+ rels_hash.each do |name, rels|
10
+ rels&.compact&.each_slice(@config.limit.to_i) do |rels_batch|
11
+ save_rels_batch_to_file(name, rels_batch)
12
+ end
13
+ end
14
+ end
15
+
16
+ def save_rels_batch_to_file(name, rels_batch)
17
+ model = Model.get_model(name)
18
+
19
+ export = {}
20
+ export[:table_name] = model.table_name
21
+ export[:nullified_relationships] = rels_batch
22
+
23
+ content = JSON.pretty_generate(export)
24
+ file_name = "nullified_relationships/build_#{@file_index}.json"
25
+ @file_index += 1
26
+ file_path = @subfolder.present? ? "#{@subfolder}/#{file_name}" : file_name
27
+ save_file(file_path, content)
28
+ end
29
+ end
data/lib/config.rb CHANGED
@@ -2,7 +2,7 @@
2
2
  require 'optparse'
3
3
 
4
4
  class Config
5
- attr_reader :if_backup,
5
+ attr_accessor :if_backup,
6
6
  :dry_run,
7
7
  :limit,
8
8
  :threshold,
@@ -13,7 +13,10 @@ class Config
13
13
  :org_id,
14
14
  :move_logs,
15
15
  :remove_orphans,
16
- :destination_db_url
16
+ :orphans_table,
17
+ :destination_db_url,
18
+ :load_from_files,
19
+ :id_gap
17
20
 
18
21
  def initialize(args={}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
19
22
  set_values(args)
@@ -96,18 +99,40 @@ class Config
96
99
  config.dig('backup', 'remove_orphans'),
97
100
  false
98
101
  )
102
+ @orphans_table = first_not_nil(
103
+ args[:orphans_table],
104
+ argv_opts[:orphans_table],
105
+ ENV['BACKUP_ORPHANS_TABLE'],
106
+ config.dig('backup', 'orphans_table'),
107
+ false
108
+ )
99
109
  @destination_db_url = first_not_nil(
100
110
  args[:destination_db_url],
101
111
  argv_opts[:destination_db_url],
102
112
  ENV['BACKUP_DESTINATION_DB_URL'],
103
113
  connection_details.dig(ENV['RAILS_ENV'], 'destination')
104
114
  )
115
+ @load_from_files = first_not_nil(
116
+ args[:load_from_files],
117
+ argv_opts[:load_from_files],
118
+ ENV['BACKUP_LOAD_FROM_FILES'],
119
+ config.dig('backup', 'load_from_files'),
120
+ false
121
+ )
122
+ @id_gap = first_not_nil(
123
+ args[:id_gap],
124
+ argv_opts[:id_gap],
125
+ ENV['BACKUP_ID_GAP'],
126
+ config.dig('backup', 'id_gap'),
127
+ 1000
128
+ )
105
129
  end
106
130
 
107
131
  def check_values
108
- if !@move_logs && !@remove_orphans && !@threshold
132
+ if !@move_logs && !@remove_orphans && !@threshold && !@user_id && !@org_id && !@repo_id && !@load_from_files
109
133
  message = abort_message("Please provide the threshold argument. Data younger than it will be omitted. " +
110
- "Threshold defines number of months from now.")
134
+ "Threshold defines number of months from now. Alternatively you can define user_id, org_id or repo_id " +
135
+ "to remove whole user, organization or repository with all dependencies.")
111
136
  abort message
112
137
  end
113
138
 
@@ -126,10 +151,12 @@ class Config
126
151
  end
127
152
 
128
153
  def abort_message(intro)
129
- "\n#{intro} Example usage:\n"+
130
- "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6\n" +
154
+ "\n#{intro}\n\nExample usage:\n"+
155
+ "\n $ bin/travis_backup 'postgres://my_database_url' --threshold 6" +
156
+ "\n $ bin/travis_backup 'postgres://my_database_url' --user_id 1\n" +
131
157
  "\nor using in code:\n" +
132
- "\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)\n" +
158
+ "\n Backup.new(database_url: 'postgres://my_database_url', threshold: 6)" +
159
+ "\n Backup.new(database_url: 'postgres://my_database_url', user_id: 1)\n" +
133
160
  "\nYou can also set it using environment variables or configuration files.\n"
134
161
  end
135
162
 
@@ -147,7 +174,10 @@ class Config
147
174
  opt.on('-o', '--org_id X') { |o| options[:org_id] = o.to_i }
148
175
  opt.on('--move_logs') { |o| options[:move_logs] = o }
149
176
  opt.on('--remove_orphans') { |o| options[:remove_orphans] = o }
177
+ opt.on('--orphans_table X') { |o| options[:orphans_table] = o }
150
178
  opt.on('--destination_db_url X') { |o| options[:destination_db_url] = o }
179
+ opt.on('--load_from_files') { |o| options[:load_from_files] = o }
180
+ opt.on('--id_gap X') { |o| options[:id_gap] = o.to_i }
151
181
  end.parse!
152
182
 
153
183
  options[:database_url] = ARGV.shift if ARGV[0]
@@ -7,10 +7,12 @@ class DryRunReporter
7
7
  @report = {}
8
8
  end
9
9
 
10
- def add_to_report(key, *values)
11
- report[key] = [] if report[key].nil?
12
- report[key].concat(values)
13
- report[key].uniq!
10
+ def add_to_report(*args)
11
+ if args.first.is_a?(Hash)
12
+ add_to_report_as_hash(args.first)
13
+ else
14
+ add_to_report_as_key_and_values(*args)
15
+ end
14
16
  end
15
17
 
16
18
  def print_report
@@ -27,6 +29,18 @@ class DryRunReporter
27
29
 
28
30
  private
29
31
 
32
+ def add_to_report_as_key_and_values(key, *values)
33
+ report[key] = [] if report[key].nil?
34
+ report[key].concat(values)
35
+ report[key].uniq!
36
+ end
37
+
38
+ def add_to_report_as_hash(hash)
39
+ hash.each do |key, array|
40
+ add_to_report_as_key_and_values(key, *array)
41
+ end
42
+ end
43
+
30
44
  def print_report_line(symbol)
31
45
  puts " - #{symbol}: #{@report[symbol].to_json}" if @report[symbol].any?
32
46
  end
data/lib/id_hash.rb ADDED
@@ -0,0 +1,97 @@
1
+ require 'model'
2
+
3
+ class HashOfArrays < Hash
4
+ def initialize(hash = {})
5
+ hash.each do |key, array|
6
+ self[key] = hash[key].clone
7
+ end
8
+ end
9
+
10
+ def clone
11
+ self.class.new(self)
12
+ end
13
+
14
+ def subtract(hash)
15
+ result = self.clone
16
+ hash.each do |key, array|
17
+ next if result[key].nil?
18
+
19
+ array.each do |el|
20
+ result[key].delete(el)
21
+ end
22
+
23
+ result.delete(key) if result[key].empty?
24
+ end
25
+ result
26
+ end
27
+
28
+ def add(key, *values)
29
+ self[key] = [] if self[key].nil?
30
+ self[key].concat(values)
31
+ end
32
+
33
+ def self.join(*hashes)
34
+ result = self.new
35
+ result.join(*hashes)
36
+ end
37
+
38
+ def join(*hashes)
39
+ hashes.each do |hash|
40
+ hash.each do |key, array|
41
+ self.add(key, *array)
42
+ end
43
+ end
44
+ self
45
+ end
46
+
47
+ def sort_arrays!
48
+ self.each do |key, array|
49
+ array.sort!
50
+ end
51
+ end
52
+ end
53
+
54
+ class IdHash < HashOfArrays
55
+ def add(key, *values)
56
+ super(key, *values)
57
+ self[key].uniq!
58
+ self
59
+ end
60
+
61
+ def remove_entries_from_db(as_first: [], as_last: [])
62
+ exceptionals = as_first + as_last
63
+ remove_from_exceptional(as_first)
64
+
65
+ self.each do |name, ids|
66
+ next if exceptionals.include?(name)
67
+ remove_entries_from_array(name, ids)
68
+ end
69
+
70
+ remove_from_exceptional(as_last)
71
+ end
72
+
73
+ def with_table_symbols
74
+ result = HashOfArrays.new
75
+
76
+ self.each do |name, ids|
77
+ symbol = Model.get_model(name).table_name.to_sym
78
+ result[symbol] = ids
79
+ end
80
+
81
+ result
82
+ end
83
+
84
+ private
85
+
86
+ def remove_from_exceptional(array)
87
+ array.each do |name|
88
+ ids = self[name]
89
+ remove_entries_from_array(name, ids)
90
+ end
91
+ end
92
+
93
+ def remove_entries_from_array(model_name, ids)
94
+ model = Model.get_model(model_name)
95
+ model.delete(ids) if model.present?
96
+ end
97
+ end