backup 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,47 +6,73 @@ module Backup
6
6
  include Backup::Utilities::Helpers
7
7
  include Backup::Configuration::Helpers
8
8
 
9
- ##
10
- # Creates a new instance of the MongoDB database object
11
- # * Called using super(model) from subclasses *
12
- def initialize(model)
9
+ attr_reader :model, :database_id, :dump_path
10
+
11
+ def initialize(model, database_id = nil)
13
12
  @model = model
13
+ @database_id = database_id.to_s.gsub(/\W/, '_') if database_id
14
+ @dump_path = File.join(Config.tmp_path, model.trigger, 'databases')
14
15
  load_defaults!
15
16
  end
16
17
 
17
- ##
18
- # Super method for all child (database) objects. Every database object's #perform!
19
- # method should call #super before anything else to prepare
20
18
  def perform!
19
+ log!(:started)
21
20
  prepare!
22
- log!
23
21
  end
24
22
 
25
23
  private
26
24
 
27
- ##
28
- # Defines the @dump_path and ensures it exists by creating it
29
25
  def prepare!
30
- @dump_path = File.join(
31
- Config.tmp_path,
32
- @model.trigger,
33
- 'databases',
34
- self.class.name.split('::').last
35
- )
36
- FileUtils.mkdir_p(@dump_path)
26
+ FileUtils.mkdir_p(dump_path)
37
27
  end
38
28
 
39
29
  ##
40
- # Return the database name, with Backup namespace removed
30
+ # Sets the base filename for the final dump file to be saved in +dump_path+,
31
+ # based on the class name. e.g. databases/MySQL.sql
32
+ #
33
+ # +database_id+ will be appended if it is defined.
34
+ # e.g. databases/MySQL-database_id.sql
35
+ #
36
+ # If multiple Databases of the same class are defined and no +database_id+
37
+ # is defined, the user will be warned and one will be auto-generated.
38
+ #
39
+ # Model#initialize calls this method *after* all defined databases have
40
+ # been initialized so `backup perform --check` can report these warnings.
41
+ def dump_filename
42
+ @dump_filename ||= begin
43
+ unless database_id
44
+ if model.databases.select {|d| d.class == self.class }.count > 1
45
+ sleep 1; @database_id = Time.now.to_i.to_s[-5, 5]
46
+ Logger.warn Errors::Database::ConfigurationError.new(<<-EOS)
47
+ Database Identifier Missing
48
+ When multiple Databases are configured in a single Backup Model
49
+ that have the same class (MySQL, PostgreSQL, etc.), the optional
50
+ +database_id+ must be specified to uniquely identify each instance.
51
+ e.g. database MySQL, :database_id do |db|
52
+ This will result in an output file in your final backup package like:
53
+ databases/MySQL-database_id.sql
54
+
55
+ Backup has auto-generated an identifier (#{ database_id }) for this
56
+ database dump and will now continue.
57
+ EOS
58
+ end
59
+ end
60
+
61
+ self.class.name.split('::').last + (database_id ? "-#{ database_id }" : '')
62
+ end
63
+ end
64
+
41
65
  def database_name
42
- self.class.to_s.sub('Backup::', '')
66
+ @database_name ||= self.class.to_s.sub('Backup::', '') +
67
+ (database_id ? " (#{ database_id })" : '')
43
68
  end
44
69
 
45
- ##
46
- # Logs a message to the console and log file to inform
47
- # the client that Backup is dumping the database
48
- def log!
49
- Logger.info "#{ database_name } started dumping and archiving '#{ name }'."
70
+ def log!(action)
71
+ msg = case action
72
+ when :started then 'Started...'
73
+ when :finished then 'Finished!'
74
+ end
75
+ Logger.info "#{ database_name } #{ msg }"
50
76
  end
51
77
  end
52
78
  end
@@ -29,202 +29,176 @@ module Backup
29
29
  attr_accessor :additional_options
30
30
 
31
31
  ##
32
- # Path to the mongodump utility (optional)
33
- attr_accessor :mongodump_utility
34
-
35
- attr_deprecate :utility_path, :version => '3.0.21',
36
- :message => 'Use MongoDB#mongodump_utility instead.',
37
- :action => lambda {|klass, val| klass.mongodump_utility = val }
38
-
39
- ##
40
- # Path to the mongo utility (optional)
41
- attr_accessor :mongo_utility
42
-
43
- ##
44
- # 'lock' dump meaning wrapping mongodump with fsync & lock
32
+ # Forces mongod to flush all pending write operations to the disk and
33
+ # locks the entire mongod instance to prevent additional writes until the
34
+ # dump is complete.
35
+ #
36
+ # Note that if Profiling is enabled, this will disable it and will not
37
+ # re-enable it after the dump is complete.
45
38
  attr_accessor :lock
46
39
 
47
40
  ##
48
- # Creates a new instance of the MongoDB database object
49
- def initialize(model, &block)
50
- super(model)
51
-
52
- @only_collections ||= Array.new
53
- @additional_options ||= Array.new
54
- @ipv6 ||= false
55
- @lock ||= false
41
+ # Creates a dump of the database that includes an oplog, to create a
42
+ # point-in-time snapshot of the state of a mongod instance.
43
+ #
44
+ # If this option is used, you would not use the `lock` option.
45
+ #
46
+ # This will only work against nodes that maintain a oplog.
47
+ # This includes all members of a replica set, as well as master nodes in
48
+ # master/slave replication deployments.
49
+ attr_accessor :oplog
56
50
 
51
+ def initialize(model, database_id = nil, &block)
52
+ super
57
53
  instance_eval(&block) if block_given?
58
-
59
- @mongodump_utility ||= utility(:mongodump)
60
- @mongo_utility ||= utility(:mongo)
61
54
  end
62
55
 
63
- ##
64
- # Performs the mongodump command and outputs the data to the
65
- # specified path based on the 'trigger'. If the user hasn't specified any
66
- # specific collections to dump, it'll dump everything. If the user has specified
67
- # collections to dump, it'll loop through the array of collections and invoke the
68
- # 'mongodump' command once per collection
69
56
  def perform!
70
57
  super
71
58
 
72
59
  lock_database if @lock
73
- @only_collections.empty? ? dump! : specific_collection_dump!
60
+ dump!
61
+ package!
74
62
 
75
- rescue => err
76
- raise Errors::Database::MongoDBError.wrap(err, 'Database Dump Failed!')
77
63
  ensure
78
64
  unlock_database if @lock
79
- package! unless err
80
65
  end
81
66
 
82
67
  private
83
68
 
84
69
  ##
85
- # Builds and runs the mongodump command
70
+ # Performs all required mongodump commands, dumping the output files
71
+ # into the +dump_packaging_path+ directory for packaging.
86
72
  def dump!
87
- run(mongodump)
88
- end
73
+ FileUtils.mkdir_p dump_packaging_path
89
74
 
90
- ##
91
- # For each collection in the @only_collections array, it'll
92
- # build the whole 'mongodump' command, append the '--collection' option,
93
- # and run the command built command
94
- def specific_collection_dump!
95
- @only_collections.each do |collection|
96
- run("#{mongodump} --collection='#{collection}'")
75
+ collections = Array(only_collections)
76
+ if collections.empty?
77
+ run(mongodump)
78
+ else
79
+ collections.each do |collection|
80
+ run("#{ mongodump } --collection='#{ collection }'")
81
+ end
97
82
  end
98
83
  end
99
84
 
100
85
  ##
101
- # Builds the full mongodump string based on all attributes
102
- def mongodump
103
- "#{ mongodump_utility } #{ database } #{ credential_options } " +
104
- "#{ connectivity_options } #{ ipv6_option } #{ user_options } #{ dump_directory }"
105
- end
106
-
107
- ##
108
- # If a compressor is configured, packages the dump_path into a
109
- # single compressed tar archive, then removes the dump_path.
110
- # e.g.
111
- # if the database was dumped to:
112
- # ~/Backup/.tmp/databases/MongoDB/
113
- # then it will be packaged into:
114
- # ~/Backup/.tmp/databases/MongoDB-<timestamp>.tar.gz
86
+ # Creates a tar archive of the +dump_packaging_path+ directory
87
+ # and stores it in the +dump_path+ using +dump_filename+.
88
+ #
89
+ # <trigger>/databases/MongoDB[-<database_id>].tar[.gz]
90
+ #
91
+ # If successful, +dump_packaging_path+ is removed.
115
92
  def package!
116
- return unless @model.compressor
93
+ pipeline = Pipeline.new
94
+ dump_ext = 'tar'
117
95
 
118
- pipeline = Pipeline.new
119
- base_dir = File.dirname(@dump_path)
120
- dump_dir = File.basename(@dump_path)
121
- timestamp = Time.now.to_i.to_s[-5, 5]
122
- outfile = @dump_path + '-' + timestamp + '.tar'
96
+ pipeline << "#{ utility(:tar) } -cf - " +
97
+ "-C '#{ dump_path }' '#{ dump_filename }'"
123
98
 
124
- Logger.info(
125
- "#{ database_name } started compressing and packaging:\n" +
126
- " '#{ @dump_path }'"
127
- )
128
-
129
- pipeline << "#{ utility(:tar) } -cf - -C '#{ base_dir }' '#{ dump_dir }'"
130
- @model.compressor.compress_with do |command, ext|
99
+ model.compressor.compress_with do |command, ext|
131
100
  pipeline << command
132
- outfile << ext
133
- end
101
+ dump_ext << ext
102
+ end if model.compressor
103
+
104
+ pipeline << "#{ utility(:cat) } > " +
105
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
134
106
 
135
- pipeline << "#{ utility(:cat) } > #{ outfile }"
136
107
  pipeline.run
137
108
  if pipeline.success?
138
- Logger.info(
139
- "#{ database_name } completed compressing and packaging:\n" +
140
- " '#{ outfile }'"
141
- )
142
- FileUtils.rm_rf(@dump_path)
109
+ FileUtils.rm_rf dump_packaging_path
110
+ log!(:finished)
143
111
  else
144
112
  raise Errors::Database::PipelineError,
145
- "#{ database_name } Failed to create compressed dump package:\n" +
146
- "'#{ outfile }'\n" +
147
- pipeline.error_messages
113
+ "#{ database_name } Dump Failed!\n" + pipeline.error_messages
148
114
  end
149
115
  end
150
116
 
151
- ##
152
- # Returns the MongoDB database selector syntax
153
- def database
117
+ def dump_packaging_path
118
+ File.join(dump_path, dump_filename)
119
+ end
120
+
121
+ def mongodump
122
+ "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
123
+ "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
124
+ "#{ user_options } --out='#{ dump_packaging_path }'"
125
+ end
126
+
127
+ def name_option
154
128
  "--db='#{ name }'" if name
155
129
  end
156
130
 
157
- ##
158
- # Builds the MongoDB credentials syntax to authenticate the user
159
- # to perform the database dumping process
160
131
  def credential_options
161
- %w[username password].map do |option|
162
- next if send(option).to_s.empty?
163
- "--#{option}='#{send(option)}'"
164
- end.compact.join(' ')
132
+ opts = []
133
+ opts << "--username='#{ username }'" if username
134
+ opts << "--password='#{ password }'" if password
135
+ opts.join(' ')
165
136
  end
166
137
 
167
- ##
168
- # Builds the MongoDB connectivity options syntax to connect the user
169
- # to perform the database dumping process
170
138
  def connectivity_options
171
- %w[host port].map do |option|
172
- next if send(option).to_s.empty?
173
- "--#{option}='#{send(option)}'"
174
- end.compact.join(' ')
139
+ opts = []
140
+ opts << "--host='#{ host }'" if host
141
+ opts << "--port='#{ port }'" if port
142
+ opts.join(' ')
175
143
  end
176
144
 
177
- ##
178
- # Returns the mongodump syntax for enabling ipv6
179
145
  def ipv6_option
180
- @ipv6 ? '--ipv6' : ''
146
+ '--ipv6' if ipv6
181
147
  end
182
148
 
183
- ##
184
- # Builds a MongoDB compatible string for the
185
- # additional options specified by the user
186
- def user_options
187
- @additional_options.join(' ')
149
+ def oplog_option
150
+ '--oplog' if oplog
188
151
  end
189
152
 
190
- ##
191
- # Returns the MongoDB syntax for determining where to output all the database dumps,
192
- # e.g. ~/Backup/.tmp/databases/MongoDB/<databases here>/<database collections>
193
- def dump_directory
194
- "--out='#{ @dump_path }'"
153
+ def user_options
154
+ Array(additional_options).join(' ')
195
155
  end
196
156
 
197
- ##
198
- # Locks and FSyncs the database to bring it up to sync
199
- # and ensure no 'write operations' are performed during the
200
- # dump process
201
157
  def lock_database
202
- lock_command = <<-EOS.gsub(/^ +/, ' ')
158
+ lock_command = <<-EOS.gsub(/^ +/, '')
203
159
  echo 'use admin
204
- db.runCommand({"fsync" : 1, "lock" : 1})' | #{ "#{ mongo_utility } #{ mongo_uri }" }
160
+ db.setProfilingLevel(0)
161
+ db.fsyncLock()' | #{ mongo_shell }
205
162
  EOS
206
163
 
207
164
  run(lock_command)
208
165
  end
209
166
 
210
- ##
211
- # Unlocks the (locked) database
212
167
  def unlock_database
213
- unlock_command = <<-EOS.gsub(/^ +/, ' ')
168
+ unlock_command = <<-EOS.gsub(/^ +/, '')
214
169
  echo 'use admin
215
- db.$cmd.sys.unlock.findOne()' | #{ "#{ mongo_utility } #{ mongo_uri }" }
170
+ db.fsyncUnlock()' | #{ mongo_shell }
216
171
  EOS
217
172
 
218
173
  run(unlock_command)
219
174
  end
220
175
 
221
- ##
222
- # Builds a Mongo URI based on the provided attributes
223
- def mongo_uri
224
- ["#{ host }:#{ port }#{ ('/' + name) if name }",
225
- credential_options, ipv6_option].join(' ').strip
176
+ def mongo_shell
177
+ cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
178
+ cmd << " #{ credential_options }".rstrip
179
+ cmd << " #{ ipv6_option }".rstrip
180
+ cmd << " '#{ name }'" if name
181
+ cmd
226
182
  end
227
183
 
184
+ attr_deprecate :utility_path, :version => '3.0.21',
185
+ :message => 'Use Backup::Utilities.configure instead.',
186
+ :action => lambda {|klass, val|
187
+ Utilities.configure { mongodump val }
188
+ }
189
+
190
+ attr_deprecate :mongodump_utility, :version => '3.3.0',
191
+ :message => 'Use Backup::Utilities.configure instead.',
192
+ :action => lambda {|klass, val|
193
+ Utilities.configure { mongodump val }
194
+ }
195
+
196
+ attr_deprecate :mongo_utility, :version => '3.3.0',
197
+ :message => 'Use Backup::Utilities.configure instead.',
198
+ :action => lambda {|klass, val|
199
+ Utilities.configure { mongo val }
200
+ }
201
+
228
202
  end
229
203
  end
230
204
  end
@@ -19,42 +19,33 @@ module Backup
19
19
 
20
20
  ##
21
21
  # Tables to skip while dumping the database
22
+ #
23
+ # If `name` is set to :all (or not specified), these must include
24
+ # a database name. e.g. 'name.table'.
25
+ # If `name` is given, these may simply be table names.
22
26
  attr_accessor :skip_tables
23
27
 
24
28
  ##
25
- # Tables to dump, tables that aren't specified won't get dumped
29
+ # Tables to dump. This in only valid if `name` is specified.
30
+ # If none are given, the entire database will be dumped.
26
31
  attr_accessor :only_tables
27
32
 
28
33
  ##
29
34
  # Additional "mysqldump" options
30
35
  attr_accessor :additional_options
31
36
 
32
- ##
33
- # Path to mysqldump utility (optional)
34
- attr_accessor :mysqldump_utility
35
-
36
- attr_deprecate :utility_path, :version => '3.0.21',
37
- :message => 'Use MySQL#mysqldump_utility instead.',
38
- :action => lambda {|klass, val| klass.mysqldump_utility = val }
39
-
40
- ##
41
- # Creates a new instance of the MySQL adapter object
42
- def initialize(model, &block)
43
- super(model)
44
-
45
- @skip_tables ||= Array.new
46
- @only_tables ||= Array.new
47
- @additional_options ||= Array.new
48
-
37
+ def initialize(model, database_id = nil, &block)
38
+ super
49
39
  instance_eval(&block) if block_given?
50
40
 
51
41
  @name ||= :all
52
- @mysqldump_utility ||= utility(:mysqldump)
53
42
  end
54
43
 
55
44
  ##
56
- # Performs the mysqldump command and outputs the
57
- # data to the specified path based on the 'trigger'
45
+ # Performs the mysqldump command and outputs the dump file
46
+ # in the +dump_path+ using +dump_filename+.
47
+ #
48
+ # <trigger>/databases/MySQL[-<database_id>].sql[.gz]
58
49
  def perform!
59
50
  super
60
51
 
@@ -62,100 +53,83 @@ module Backup
62
53
  dump_ext = 'sql'
63
54
 
64
55
  pipeline << mysqldump
65
- if @model.compressor
66
- @model.compressor.compress_with do |command, ext|
67
- pipeline << command
68
- dump_ext << ext
69
- end
70
- end
56
+
57
+ model.compressor.compress_with do |command, ext|
58
+ pipeline << command
59
+ dump_ext << ext
60
+ end if model.compressor
71
61
 
72
62
  pipeline << "#{ utility(:cat) } > " +
73
- "'#{ File.join(@dump_path, dump_filename) }.#{ dump_ext }'"
63
+ "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
64
+
74
65
  pipeline.run
75
66
  if pipeline.success?
76
- Logger.info "#{ database_name } Complete!"
67
+ log!(:finished)
77
68
  else
78
69
  raise Errors::Database::PipelineError,
79
- "#{ database_name } Dump Failed!\n" +
80
- pipeline.error_messages
70
+ "#{ database_name } Dump Failed!\n" + pipeline.error_messages
81
71
  end
82
72
  end
83
73
 
84
74
  private
85
75
 
86
- ##
87
- # Builds the full mysqldump string based on all attributes
88
76
  def mysqldump
89
- "#{ mysqldump_utility } #{ credential_options } #{ connectivity_options } " +
90
- "#{ user_options } #{ db_name } #{ tables_to_dump } #{ tables_to_skip }"
77
+ "#{ utility(:mysqldump) } #{ credential_options } " +
78
+ "#{ connectivity_options } #{ user_options } #{ name_option } " +
79
+ "#{ tables_to_dump } #{ tables_to_skip }"
91
80
  end
92
81
 
93
- ##
94
- # Returns the filename to use for dumping the database(s)
95
- def dump_filename
96
- dump_all? ? 'all-databases' : name
97
- end
98
-
99
- ##
100
- # Builds the credentials MySQL syntax to authenticate the user
101
- # to perform the database dumping process
102
82
  def credential_options
103
- %w[username password].map do |option|
104
- next if send(option).to_s.empty?
105
- "--#{option}='#{send(option)}'".gsub('--username', '--user')
106
- end.compact.join(' ')
83
+ opts = []
84
+ opts << "--user='#{ username }'" if username
85
+ opts << "--password='#{ password }'" if password
86
+ opts.join(' ')
107
87
  end
108
88
 
109
- ##
110
- # Builds the MySQL connectivity options syntax to connect the user
111
- # to perform the database dumping process
112
89
  def connectivity_options
113
- %w[host port socket].map do |option|
114
- next if send(option).to_s.empty?
115
- "--#{option}='#{send(option)}'"
116
- end.compact.join(' ')
90
+ return "--socket='#{ socket }'" if socket
91
+
92
+ opts = []
93
+ opts << "--host='#{ host }'" if host
94
+ opts << "--port='#{ port }'" if port
95
+ opts.join(' ')
117
96
  end
118
97
 
119
- ##
120
- # Builds a MySQL compatible string for the additional options
121
- # specified by the user
122
98
  def user_options
123
- additional_options.join(' ')
99
+ Array(additional_options).join(' ')
124
100
  end
125
101
 
126
- ##
127
- # Returns the database name to use in the mysqldump command.
128
- # When dumping all databases, the database name is replaced
129
- # with the command option to dump all databases.
130
- def db_name
102
+ def name_option
131
103
  dump_all? ? '--all-databases' : name
132
104
  end
133
105
 
134
- ##
135
- # Builds the MySQL syntax for specifying which tables to dump
136
- # during the dumping of the database
137
106
  def tables_to_dump
138
- only_tables.join(' ') unless dump_all?
107
+ Array(only_tables).join(' ') unless dump_all?
139
108
  end
140
109
 
141
- ##
142
- # Builds the MySQL syntax for specifying which tables to skip
143
- # during the dumping of the database
144
110
  def tables_to_skip
145
- skip_tables.map do |table|
111
+ Array(skip_tables).map do |table|
146
112
  table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
147
113
  "--ignore-table='#{ table }'"
148
114
  end.join(' ')
149
115
  end
150
116
 
151
- ##
152
- # Return true if we're dumping all databases.
153
- # `name` will be set to :all if it is not set,
154
- # so this will be true by default
155
117
  def dump_all?
156
118
  name == :all
157
119
  end
158
120
 
121
+ attr_deprecate :utility_path, :version => '3.0.21',
122
+ :message => 'Use Backup::Utilities.configure instead.',
123
+ :action => lambda {|klass, val|
124
+ Utilities.configure { mysqldump val }
125
+ }
126
+
127
+ attr_deprecate :mysqldump_utility, :version => '3.3.0',
128
+ :message => 'Use Backup::Utilities.configure instead.',
129
+ :action => lambda {|klass, val|
130
+ Utilities.configure { mysqldump val }
131
+ }
132
+
159
133
  end
160
134
  end
161
135
  end