blackstack-deployer 1.2.12 → 1.2.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. checksums.yaml +4 -4
  2. data/lib/blackstack-deployer.rb +721 -669
  3. metadata +5 -5
@@ -1,670 +1,722 @@
1
- require 'blackstack-nodes'
2
- require 'sequel'
3
-
4
- module BlackStack
5
- # Deployer is a library that can be used to deploy a cluster of nodes.
6
- module Deployer
7
- @@logger = BlackStack::BaseLogger.new(nil)
8
- @@nodes = []
9
- @@routines = []
10
-
11
- # set the logger
12
- def self.set_logger(i_logger)
13
- @@logger = i_logger
14
- end
15
-
16
- # get the logger assigned to the module
17
- def self.logger
18
- @@logger
19
- end # def self.errors
20
-
21
- # get the array of nodes assigned to the module
22
- def self.nodes
23
- @@nodes
24
- end # def self.nodes
25
-
26
- # get the array of routines assigned to the module
27
- def self.routines
28
- @@routines
29
- end # def self.routines
30
-
31
- # inherit BlackStack::Infrastructure::NodeModule, including features of deployer.
32
- module NodeModule
33
- attr_accessor :deployment_routine, :parameters
34
-
35
- include BlackStack::Infrastructure::NodeModule
36
-
37
- # get the IP address for an interface using the ip addr command.
38
- # this is a helper method for installing cockroachdb nodes.
39
- def eth0_ip(interface='eth0')
40
- a = self.ssh.exec!('ip addr show dev eth0').scan(/inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
41
- return nil if a.size == 0
42
- return a.last.to_s.gsub(/inet /, '')
43
- end
44
-
45
- def self.descriptor_errors(h)
46
- errors = BlackStack::Infrastructure::NodeModule.descriptor_errors(h)
47
-
48
- # validate: does not exist any other element in @@nodes with the same value for the parameter h[:name]
49
- errors << "The parameter h[:name] is not unique" if BlackStack::Deployer.nodes.select{|n| n.name == h[:name]}.length > 0
50
-
51
- # validate: h[:deployment_routine] is not nil
52
- errors << "The parameter h[:deployment_routine] is required" if h[:deployment_routine].nil?
53
-
54
- # validate: h[:deployment_routine] is a string
55
- errors << "The parameter h[:deployment_routine] is not a string" unless h[:deployment_routine].is_a?(String)
56
-
57
- # return list of errors
58
- errors.uniq
59
- end
60
-
61
- def initialize(h, i_logger=nil)
62
- self.parameters = h
63
- errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
64
- raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
65
- super(h, i_logger)
66
- self.deployment_routine = h[:deployment_routine]
67
- end # def self.create(h)
68
-
69
- def to_hash
70
- h = super
71
- h[:deployment_routine] = self.deployment_routine
72
- h
73
- end # def to_hash
74
-
75
- def deploy()
76
- BlackStack::Deployer::run_routine(self.name, self.deployment_routine);
77
- end
78
- end # module NodeModule
79
-
80
- # define attributes and methods of a deployer routine
81
- module RoutineModule
82
- attr_accessor :name, :commands
83
-
84
- def self.descriptor_errors(h)
85
- errors = []
86
-
87
- # validate: the parameter h is a hash
88
- errors << "The parameter h is not a hash" unless h.is_a?(Hash)
89
-
90
- # validate: the paramerer h has a key :name
91
- errors << "The parameter h does not have a key :name" unless h.has_key?(:name)
92
-
93
- # validate: the paramerer h has a key :command
94
- errors << "The parameter h does not have a key :commands" unless h.has_key?(:commands)
95
-
96
- # validate: the parameter h[:name] is a string or a symbol
97
- errors << "The parameter h[:name] is not a string" unless h[:name].is_a?(String)
98
-
99
- # validate: the parameter h[:name] is not 'reboot' because it is a reserved name
100
- errors << "The parameter h[:name] is a reserved name (#{h[:name].to_s})" if h[:name] == 'reboot'
101
-
102
- # validate: the parameter h[:commands] is required
103
- errors << "The parameter h[:commands] is required" if h[:commands].nil?
104
-
105
- # validate: the parametrer h[:commands] is an array
106
- errors << "The parameter h[:commands] is not an array" unless h[:commands].is_a?(Array)
107
-
108
- # validate: the parameter h[:commands] has at least one element
109
- errors << "The parameter h[:commands] does not have at least one element" unless h[:commands].size > 0
110
-
111
- # validate: each element of the array h[:commands] is a hash
112
- h[:commands].each do |c|
113
- errors += BlackStack::Deployer::CommandModule.descriptor_errors(c)
114
- end # h[:commands].each do |c|
115
-
116
- errors.uniq
117
- end # def self.descriptor_error(h)
118
-
119
- def initialize(h)
120
- errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
121
- raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
122
- self.name = h[:name]
123
- self.commands = []
124
- h[:commands].each do |c|
125
- self.commands << BlackStack::Deployer::Command.new(c)
126
- end
127
- end
128
-
129
- def to_hash
130
- h = {}
131
- h[:name] = self.name
132
- h[:commands] = []
133
- self.commands.each do |c|
134
- h[:commands] << c.to_hash
135
- end
136
- h
137
- end
138
-
139
- def run(node)
140
- ret = []
141
- self.commands.each do |c|
142
- BlackStack::Deployer.logger.logs "Running command: #{c.command.to_s}... "
143
- h = c.run(node)
144
- ret << h
145
-
146
- #BlackStack::Deployer.logger.logs "Result: "
147
- #BlackStack::Deployer.logger.logf h.to_s
148
-
149
- if h[:errors].size == 0
150
- BlackStack::Deployer.logger.done
151
- else
152
- BlackStack::Deployer.logger.logf('error: ' + h.to_s)
153
- raise "Error running command: #{h.to_s}"
154
- end
155
- end
156
- ret
157
- end # def run(node)
158
- end # module RoutineModule
159
-
160
- # define attributes and methods of a routine's command
161
- module CommandModule
162
- attr_accessor :command, :matches, :nomatches, :sudo
163
-
164
- def self.descriptor_errors(c)
165
- errors = []
166
-
167
- # validate: h is a hash
168
- errors << "The command descriptor is not a hash" unless c.is_a?(Hash)
169
-
170
- # validate: the hash c has a key :command
171
- errors << "The command descriptor does not have a key :command" unless c.has_key?(:command)
172
-
173
- # validate: the value of c[:command] is a string or symbol
174
- errors << "The value of c[:command] is not a string and is not a symbol" unless c[:command].is_a?(String) || c[:command].is_a?(Symbol)
175
-
176
- # validate: if the key :sudo exists, then its value is a boolean
177
- if c.has_key?(:sudo)
178
- errors << "The value of c[:sudo] is not a boolean" unless c[:sudo].is_a?(TrueClass) || c[:sudo].is_a?(FalseClass)
179
- end
180
-
181
- # if the parameter h[:name] is a symbol
182
- if c[:command].is_a?(Symbol)
183
- if c[:command] == :reboot
184
- # :reboot is a reserved word, so it is fine to call :reboot
185
- else
186
- # validate: existis a routine with a the value c[:command].to_s on its :name key
187
- errors << "The routine with the name #{c[:command].to_s} does not exist" unless BlackStack::Deployer::routines.select { |r| r.name == c[:command].to_s }.size > 0
188
- end
189
- end
190
-
191
- # if c[:matches] exists
192
- if c.has_key?(:matches)
193
- # validate: the value of c[:matches] must by a regex or an array
194
- errors << "The value of the key :matches is not a regex nor an array" unless c[:matches].is_a?(Regexp) || c[:matches].is_a?(Array)
195
- # if c[:matches] is a array
196
- if c[:matches].is_a?(Array)
197
- # validate: each element in the the array c[:matches] is a regex
198
- c[:matches].each do |m|
199
- errors += BlackStack::Deployer::MatchModule.descriptor_errors(m)
200
- end # each
201
- end # if c[:matches].is_a?(Array)
202
- end # if :matches exists
203
-
204
- # if c[:nomatches] exists
205
- if c.has_key?(:nomatches)
206
- # validate: the value of c[:nomatches] must by a regex or an array
207
- errors << "The value of the key :nomatches is not a regex nor an array" unless c[:nomatches].is_a?(Regexp) || c[:nomatches].is_a?(Array)
208
- # if c[:nomatches] is a array
209
- if c[:nomatches].is_a?(Array)
210
- # validate: each element in the the array c[:nomatches] is a hash
211
- c[:nomatches].each do |m|
212
- errors += BlackStack::Deployer::NoMatchModule.descriptor_errors(m)
213
- end # each
214
- end # if c[:matches].is_a?(Array)
215
- end # if :matches exists
216
- #
217
- errors.uniq
218
- end # def self.descriptor_error(h)
219
-
220
- def initialize(h)
221
- errors = BlackStack::Deployer::CommandModule.descriptor_errors(h)
222
- raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
223
- self.command = h[:command]
224
- self.sudo = h[:sudo].nil? ? true : h[:sudo]
225
- self.matches = []
226
- self.nomatches = []
227
- if h.has_key?(:matches)
228
- if h[:matches].is_a?(Regexp)
229
- self.matches << BlackStack::Deployer::Match.new(h[:matches])
230
- else
231
- h[:matches].each do |m|
232
- self.matches << BlackStack::Deployer::Match.new(m)
233
- end
234
- end
235
- end
236
- if h.has_key?(:nomatches)
237
- if h[:nomatches].is_a?(Regexp)
238
- self.nomatches << BlackStack::Deployer::NoMatch.new(h[:nomatches])
239
- else
240
- h[:nomatches].each do |m|
241
- self.nomatches << BlackStack::Deployer::NoMatch.new(m)
242
- end
243
- end
244
- end
245
- end # def initialize(h)
246
-
247
- def to_hash
248
- h = {}
249
- h[:command] = self.command
250
- h[:sudo] = self.sudo
251
- h[:matches] = []
252
- h[:nomatches] = []
253
- self.matches.each do |m|
254
- h[:matches] << m.to_hash
255
- end
256
- self.nomatches.each do |m|
257
- h[:nomatches] << m.to_hash
258
- end
259
- h
260
- end # def to_hash
261
-
262
- def run(node)
263
- errors = []
264
- code = self.command
265
- output = nil
266
-
267
- # if code is a symbol
268
- if code.is_a?(Symbol)
269
-
270
- # if code is equel than :reboot
271
- if code == :reboot
272
- # call the node reboot method
273
- node.reboot
274
- else
275
- # look for a routine with this name
276
- r = BlackStack::Deployer.routines.select { |r| r.name == code.to_s }.first
277
- if !r.nil?
278
- r.run(node)
279
- else
280
- raise "The routine #{code.to_s} does not exist"
281
- end
282
- end
283
-
284
- # if code is a string
285
- elsif code.is_a?(String)
286
- # replacing parameters
287
- code.scan(/%[a-zA-Z0-9\_]+%/).each do |p|
288
- if p == '%eth0_ip%' # reserved parameter
289
- # TODO: move the method eth0_ip to the blackstack-nodes library
290
- code.gsub!(p, node.eth0_ip)
291
- elsif p == '%timestamp%' # reserved parameter
292
- # TODO: move this to a timestamp function on blackstack-core
293
- code.gsub!(p, Time.now.to_s.gsub(/\D/, ''))
294
- else
295
- if node.parameters.has_key?(p.gsub(/%/, '').to_sym)
296
- code.gsub!(p, node.parameters[p.gsub(/%/, '').to_sym].to_s)
297
- else
298
- raise "The parameter #{p} does not exist in the node descriptor #{node.parameters.to_s}"
299
- end
300
- end
301
- end
302
-
303
- #puts
304
- #puts
305
- #puts code
306
- #exit(0)
307
- #puts
308
- #puts "SUDO: #{self.sudo}"
309
- # running the command
310
- output = node.exec(code, self.sudo)
311
- #puts
312
- #puts '1'
313
- # validation: at least one of the matches should happen
314
- if self.matches.size > 0
315
- i = 0
316
- self.matches.each do |m|
317
- if m.validate(output).size == 0 # no errors
318
- i += 1
319
- end
320
- end
321
- errors << "Command output doesn't match with any of the :matches" if i == 0
322
- end # if self.matches.size > 0
323
-
324
- # validation: no one of the nomatches should happen
325
- self.nomatches.each do |m|
326
- errors += m.validate(output)
327
- end
328
- end # elsif code.is_a?(String)
329
-
330
- # return a hash descriptor of the command result
331
- {
332
- :command => self.command,
333
- :code => code,
334
- :output => output,
335
- :errors => errors,
336
- }
337
- end # def run(node)
338
- end # module CommandModule
339
-
340
- # define attributes and methods of a command's match
341
- module MatchModule
342
- attr_accessor :match
343
-
344
- def self.descriptor_errors(m)
345
- errors = []
346
- # validate the match is a regular expresion
347
- errors << "The match is not a regex" unless m.is_a?(Regexp)
348
- #
349
- errors.uniq
350
- end # def self.descriptor_error(h)
351
-
352
- def initialize(h)
353
- errors = BlackStack::Deployer::MatchModule.descriptor_errors(h)
354
- raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
355
- self.match = h
356
- end
357
-
358
- def to_hash
359
- h = {}
360
- h[:match] = self.match
361
- h
362
- end
363
-
364
- def validate(output)
365
- errors = []
366
- errors << "The output of the command does not match the regular expression #{self.match.inspect}" unless output.match(self.match)
367
- errors
368
- end
369
- end # module MatchModule
370
-
371
- # define attributes and methods of a command's nomatch
372
- module NoMatchModule
373
- attr_accessor :nomatch, :error_description
374
-
375
- def self.descriptor_errors(m)
376
- errors = []
377
- # validate the nomatch is a regular expresion
378
- errors << "Each nomatch is not a regex nor a hash" unless m.is_a?(Hash) || m.is_a?(Regexp)
379
- # if m is a hash
380
- if m.is_a?(Hash)
381
- # validate: the hash m has a key :nomatch
382
- errors << "The hash descriptor of the nomatch does not have a key :nomatch" unless m.has_key?(:nomatch)
383
- # validate: the value of m[:nomatch] is a string
384
- errors << "The value of the key :nomatch is not a regex" unless m[:nomatch].is_a?(Regexp)
385
- # validate: the hash m has a key :error_description
386
- errors << "The hash descriptor of the nomatch does not have a key :error_description" unless m.has_key?(:error_description)
387
- # validate: the value of m[:error_description] is a string
388
- errors << "The value of the key :error_description is not a string" unless m[:error_description].is_a?(String)
389
- end # if m.is_a?(Hash)
390
- #
391
- errors.uniq
392
- end # def self.descriptor_error(h)
393
-
394
- def initialize(h)
395
- errors = BlackStack::Deployer::NoMatchModule.descriptor_errors(h)
396
- raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
397
- self.nomatch = h[:nomatch]
398
- self.error_description = h[:error_description]
399
- end
400
-
401
- def to_hash
402
- h = {}
403
- h[:nomatch] = self.nomatch
404
- h[:error_description] = self.error_description
405
- h
406
- end
407
-
408
- def validate(output)
409
- errors = []
410
- if !self.error_description.nil?
411
- errors << self.error_description if output.match(self.nomatch)
412
- else
413
- errors << "The output of the command matches the regular expression #{self.nomatch.inspect}" if output.match(self.nomatch)
414
- end
415
- errors
416
- end
417
-
418
- end # module NoMatchModule
419
-
420
-
421
- # TODO: declare these classes (stub and skeleton) using blackstack-rpc
422
- #
423
- # Stub Classes
424
- # These classes represents a node, without using connection to the database.
425
- # Use this class at the client side.
426
- class Node
427
- include BlackStack::Deployer::NodeModule
428
- end # class Node
429
-
430
- class Command
431
- include BlackStack::Deployer::CommandModule
432
- end # class Command
433
-
434
- class Routine
435
- include BlackStack::Deployer::RoutineModule
436
- end # class Routine
437
-
438
- class Match
439
- include BlackStack::Deployer::MatchModule
440
- end # class Match
441
-
442
- class NoMatch
443
- include BlackStack::Deployer::NoMatchModule
444
- end # class NoMatch
445
-
446
- # add a node to the list of nodes.
447
- def self.add_node(h)
448
- errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
449
- raise errors.join(".\n") unless errors.empty?
450
- @@nodes << BlackStack::Deployer::Node.new(h, @@logger)
451
- end # def
452
-
453
- # add an array of nodes to the list of nodes.
454
- def self.add_nodes(a)
455
- # validate: the parameter a is an array
456
- raise "The parameter a is not an array" unless a.is_a?(Array)
457
- a.each { |h| BlackStack::Deployer.add_node(h) }
458
- end # def
459
-
460
- # remove all exisiting nodes in he list of nodes.
461
- # then, add the nodes in the parameter a to the list of nodes.
462
- def self.set_nodes(a)
463
- @@nodes.clear
464
- BlackStack::Deployer.add_nodes(a)
465
- end # def
466
-
467
- # add a routine to the list of routines.
468
- def self.add_routine(h)
469
- errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
470
- raise errors.join(".\n") unless errors.empty?
471
- @@routines << BlackStack::Deployer::Routine.new(h)
472
- end # def
473
-
474
- # add an array of routines to the list of routines.
475
- def self.add_routines(a)
476
- # validate: the parameter a is an array
477
- raise "The parameter a is not an array" unless a.is_a?(Array)
478
- a.each { |h| BlackStack::Deployer.add_routine(h) }
479
- end # def
480
-
481
- # remove all exisiting routines in he list of routines.
482
- # then, add the routines in the parameter a to the list of routines.
483
- def self.set_routines(a)
484
- @@routines.clear
485
- BlackStack::Deployer.add_routines(a)
486
- end # def
487
-
488
- # running a routine on a node
489
- def self.run_routine(node_name, routine_name)
490
- errors = []
491
-
492
- # find the node with the value node_name in the key :name
493
- n = @@nodes.select { |n| n.name == node_name }.first
494
-
495
- # find the routine with the value routine_name in the key :name
496
- r = @@routines.select { |r| r.name == routine_name }.first
497
-
498
- # validate: the node n exists
499
- errors << "Node #{node_name} not found" unless n
500
-
501
- # validate: the routine r exists
502
- errors << "Routine #{routine_name} not found" unless r
503
-
504
- # raise exception if any error has been found
505
- raise "The routine #{routine_name} cannot be run on the node #{node_name}: #{errors.uniq.join(".\n")}" if errors.length > 0
506
-
507
- # connect the node
508
- self.logger.logs "Connecting to node #{n.name}... "
509
- n.connect
510
- self.logger.done
511
-
512
- # run the routine
513
- self.logger.logs "Running routine #{r.name}... "
514
- r.run(n)
515
- self.logger.done
516
-
517
- # disconnect the node
518
- self.logger.logs "Disconnecting from node #{n.name}... "
519
- n.disconnect
520
- self.logger.done
521
-
522
- end # def
523
-
524
- module DB
525
- LOCKFILE = './blackstack-deployer.lock'
526
- @@checkpoint = nil
527
- @@superhuser = nil
528
- @@ndb = nil
529
- @@folder = nil
530
-
531
- def self.set_checkpoint(s)
532
- @@checkpoint = s
533
- end
534
-
535
- def self.checkpoint
536
- @@checkpoint
537
- end
538
-
539
- def self.save_checkpoint(lockfilename=BlackStack::Deployer::DB::LOCKFILE)
540
- File.new(lockfilename, "w").write(@@checkpoint)
541
- end
542
-
543
- def self.load_checkpoint(lockfilename=BlackStack::Deployer::DB::LOCKFILE)
544
- if File.exists?(lockfilename)
545
- @@checkpoint = File.new(lockfilename, "r").read
546
- else
547
- @@checkpoint = nil
548
- end
549
- @@checkpoint
550
- end
551
-
552
- def self.connect(s)
553
- @@db = Sequel::connect(s)
554
- end # def
555
-
556
- def self.set_folder(s)
557
- @@folder = s
558
- end # def
559
-
560
-
561
- # Return `true` if the name of the file matches with `/\.transactions\./`, and it doesn't match with `/\.sentences\./`, and the matches with `/\.transactions\./` are no more than one.
562
- # Otherwise, return `false`.
563
- # This method should not be called directly by user code.
564
- def self.is_transactions_file?(filename)
565
- filename =~ /\.transactions\./ && filename !~ /\.sentences\./ && filename.scan(/\.transactions\./).size == 1
566
- end
567
-
568
- # Return `true` if the name of the file matches with `/\.sentences\./`, and it doesn't match with `/\.transactions\./`, and the matches with `/\.sentences\./` are no more than one.
569
- # Otherwise({, return `false`.
570
- # This method should not be called directly by user code.
571
- def self.is_sentences_file?(filename)
572
- filename =~ /\.sentences\./ && filename !~ /\.transactions\./ && filename.scan(/\.sentences\./).size == 1
573
- end
574
-
575
- # Method to process an `.sql` file with transactions code, separated by `BEGIN;` and `COMMIT;` statements.
576
- # Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
577
- # This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.tsql\./`.
578
- # This method should not be called directly by user code.
579
- def self.execute_transactions(sql)
580
- # TODO: Code Me!
581
- end # def db_execute_tsql
582
-
583
- # Method to process an `.sql` file with one sql sentence by line.
584
- # Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
585
- # This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.sentences\./`.
586
- # This method should not be called directly by user code.
587
- def self.execute_sentences(sql, chunk_size=200)
588
- tlogger = BlackStack::Deployer::logger
589
-
590
- # Fix issue: Ruby `split': invalid byte sequence in UTF-8 (ArgumentError)
591
- # Reference: https://stackoverflow.com/questions/11065962/ruby-split-invalid-byte-sequence-in-utf-8-argumenterror
592
- #
593
- # Fix issue: `PG::SyntaxError: ERROR: at or near "��truncate": syntax error`
594
- #
595
- sql.encode!('UTF-8', :invalid => :replace, :replace => '')
596
-
597
- # Remove null bytes to avoid error: `String contains null byte`
598
- # Reference: https://stackoverflow.com/questions/29320369/coping-with-string-contains-null-byte-sent-from-users
599
- sql.gsub!("\u0000", "")
600
-
601
- # Get the array of sentences
602
- tlogger.logs "Splitting the sql sentences... "
603
- sentences = sql.split(/;/i)
604
- tlogger.logf "done (#{sentences.size})"
605
-
606
- # Chunk the array into parts of chunk_size elements
607
- # Reference: https://stackoverflow.com/questions/2699584/how-to-split-chunk-a-ruby-array-into-parts-of-x-elements
608
- tlogger.logs "Bunlding the array of sentences into chunks of #{chunk_size} each... "
609
- chunks = sentences.each_slice(chunk_size).to_a
610
- tlogger.logf "done (#{chunks.size})"
611
-
612
- chunk_number = -1
613
- chunks.each { |chunk|
614
- chunk_number += 1
615
- statement = chunk.join(";\n").to_s.strip
616
- tlogger.logs "lines #{chunk_size*chunk_number+1} to #{chunk_size*chunk_number+chunk.size} of #{sentences.size}... "
617
- begin
618
- @@db.execute(statement) #if statement.to_s.strip.size > 0
619
- tlogger.done
620
- rescue => e
621
- tlogger.logf e.to_s
622
- raise "Error executing statement: #{statement}\n#{e.message}"
623
- end
624
- }
625
- tlogger.done
626
- end # def db_execute_sql_sentences_file
627
-
628
- # Run a series of `.sql` files with updates to the database.
629
- #
630
- def self.deploy(save_checkpoints=false, lockfilename=BlackStack::Deployer::DB::LOCKFILE)
631
- tlogger = BlackStack::Deployer::logger
632
- # get list of `.sql` files in the directory `sql_path`, with a name higher than `last_filename`, sorted by name.
633
- Dir.entries(@@folder).select {
634
- |filename| filename =~ /\.sql$/ && filename > @@checkpoint.to_s
635
- }.uniq.sort.each { |filename|
636
- fullfilename = "#{@@folder}/#{filename}"
637
-
638
- tlogger.logs "#{fullfilename}... "
639
- BlackStack::Deployer::DB::execute_sentences( File.open(fullfilename).read )
640
- tlogger.done
641
-
642
- tlogger.logs "Updating checkpoint... "
643
- BlackStack::Deployer::DB::set_checkpoint filename
644
- tlogger.done
645
-
646
- tlogger.logs 'Saving checkpoint... '
647
- if save_checkpoints
648
- BlackStack::Deployer::DB::save_checkpoint(lockfilename)
649
- tlogger.done
650
- else
651
- tlogger.logf 'disabled'
652
- end
653
- }
654
- end # def
655
- end # module DB
656
-
657
- # deploying all db-updates and run all routines on all nodes
658
- def self.deploy()
659
- tlogger = BlackStack::Deployer::logger
660
-
661
- @@nodes.each { |n|
662
- tlogger.logs "Node #{n.name}... "
663
- n.deploy()
664
- tlogger.done
665
- }
666
- end # def
667
-
668
- end # module Deployer
669
-
1
+ require 'blackstack-nodes'
2
+ require 'sequel'
3
+
4
+ module BlackStack
5
+ # Deployer is a library that can be used to deploy a cluster of nodes.
6
+ module Deployer
7
+ @@logger = BlackStack::BaseLogger.new(nil)
8
+ @@show_output = false
9
+ @@nodes = []
10
+ @@routines = []
11
+
12
+ # set show_output
13
+ def self.set_show_output(value)
14
+ @@show_output = value
15
+ end
16
+
17
+ # set the logger
18
+ def self.set_logger(i_logger)
19
+ @@logger = i_logger
20
+ end
21
+
22
+ # get show_output
23
+ def self.show_output
24
+ @@show_output
25
+ end
26
+
27
+ # get the logger assigned to the module
28
+ def self.logger
29
+ @@logger
30
+ end # def self.errors
31
+
32
+ # get the array of nodes assigned to the module
33
+ def self.nodes
34
+ @@nodes
35
+ end # def self.nodes
36
+
37
+ # get the array of routines assigned to the module
38
+ def self.routines
39
+ @@routines
40
+ end # def self.routines
41
+
42
+ # inherit BlackStack::Infrastructure::NodeModule, including features of deployer.
43
+ module NodeModule
44
+ attr_accessor :deployment_routine, :parameters
45
+
46
+ include BlackStack::Infrastructure::NodeModule
47
+
48
+ def self.eth0_ip(insterface)
49
+ ret = nil
50
+ a = `ip addr show dev #{insterface}`.scan(/inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
51
+ if a.size > 0
52
+ ret = a.last.to_s.gsub(/inet /, '')
53
+ else
54
+ raise "Cannot get ip address of the interface #{insterface}"
55
+ end
56
+ ret
57
+ end
58
+
59
+ # get the IP address for an interface using the ip addr command.
60
+ # this is a helper method for installing cockroachdb nodes.
61
+ def eth0_ip()
62
+ ret = nil
63
+ a = self.ssh.exec!("ip addr show dev #{parameters[:laninterface]}").scan(/inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
64
+ if a.size > 0
65
+ ret = a.last.to_s.gsub(/inet /, '')
66
+ else
67
+ raise "Cannot get ip address of the interface #{parameters[:laninterface]}"
68
+ end
69
+ ret
70
+ end
71
+
72
+ def self.descriptor_errors(h)
73
+ errors = BlackStack::Infrastructure::NodeModule.descriptor_errors(h)
74
+
75
+ # validate: does not exist any other element in @@nodes with the same value for the parameter h[:name]
76
+ errors << "The parameter h[:name] is not unique" if BlackStack::Deployer.nodes.select{|n| n.name == h[:name]}.length > 0
77
+
78
+ # validate: h[:deployment_routine] is not nil
79
+ errors << "The parameter h[:deployment_routine] is required" if h[:deployment_routine].nil?
80
+
81
+ # validate: h[:deployment_routine] is a string
82
+ errors << "The parameter h[:deployment_routine] is not a string" unless h[:deployment_routine].is_a?(String)
83
+
84
+ # return list of errors
85
+ errors.uniq
86
+ end
87
+
88
+ def initialize(h, i_logger=nil)
89
+ self.parameters = h
90
+ errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
91
+ raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
92
+ super(h, i_logger)
93
+ self.deployment_routine = h[:deployment_routine]
94
+ end # def self.create(h)
95
+
96
+ def to_hash
97
+ h = super
98
+ h[:deployment_routine] = self.deployment_routine
99
+ h
100
+ end # def to_hash
101
+
102
+ def deploy()
103
+ BlackStack::Deployer::run_routine(self.name, self.deployment_routine);
104
+ end
105
+ end # module NodeModule
106
+
107
+ # define attributes and methods of a deployer routine
108
+ module RoutineModule
109
+ attr_accessor :name, :commands
110
+
111
+ def self.descriptor_errors(h)
112
+ errors = []
113
+
114
+ # validate: the parameter h is a hash
115
+ errors << "The parameter h is not a hash" unless h.is_a?(Hash)
116
+
117
+ # validate: the paramerer h has a key :name
118
+ errors << "The parameter h does not have a key :name" unless h.has_key?(:name)
119
+
120
+ # validate: the paramerer h has a key :command
121
+ errors << "The parameter h does not have a key :commands" unless h.has_key?(:commands)
122
+
123
+ # validate: the parameter h[:name] is a string or a symbol
124
+ errors << "The parameter h[:name] is not a string" unless h[:name].is_a?(String)
125
+
126
+ # validate: the parameter h[:name] is not 'reboot' because it is a reserved name
127
+ errors << "The parameter h[:name] is a reserved name (#{h[:name].to_s})" if h[:name] == 'reboot'
128
+
129
+ # validate: the parameter h[:commands] is required
130
+ errors << "The parameter h[:commands] is required" if h[:commands].nil?
131
+
132
+ # validate: the parametrer h[:commands] is an array
133
+ errors << "The parameter h[:commands] is not an array" unless h[:commands].is_a?(Array)
134
+
135
+ # validate: the parameter h[:commands] has at least one element
136
+ errors << "The parameter h[:commands] does not have at least one element" unless h[:commands].size > 0
137
+
138
+ # validate: each element of the array h[:commands] is a hash
139
+ h[:commands].each do |c|
140
+ errors += BlackStack::Deployer::CommandModule.descriptor_errors(c)
141
+ end # h[:commands].each do |c|
142
+
143
+ errors.uniq
144
+ end # def self.descriptor_error(h)
145
+
146
+ def initialize(h)
147
+ errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
148
+ raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
149
+ self.name = h[:name]
150
+ self.commands = []
151
+ h[:commands].each do |c|
152
+ self.commands << BlackStack::Deployer::Command.new(c)
153
+ end
154
+ end
155
+
156
+ def to_hash
157
+ h = {}
158
+ h[:name] = self.name
159
+ h[:commands] = []
160
+ self.commands.each do |c|
161
+ h[:commands] << c.to_hash
162
+ end
163
+ h
164
+ end
165
+
166
+ def run(node)
167
+ ret = []
168
+ self.commands.each do |c|
169
+ #BlackStack::Deployer.logger.logs "Running command: #{c.command.to_s}... "
170
+ h = c.run(node)
171
+ ret << h
172
+
173
+ #BlackStack::Deployer.logger.logs "Result: "
174
+ #BlackStack::Deployer.logger.logf h.to_s
175
+
176
+ if h[:errors].size == 0
177
+ #BlackStack::Deployer.logger.done
178
+ else
179
+ #BlackStack::Deployer.logger.logf('error: ' + h.to_s)
180
+ raise "Error running command:\n#{h[:errors].uniq.join("\n")}"
181
+ end
182
+ end
183
+ ret
184
+ end # def run(node)
185
+ end # module RoutineModule
186
+
187
+ # define attributes and methods of a routine's command
188
+ module CommandModule
189
+ attr_accessor :command, :matches, :nomatches, :sudo, :background
190
+
191
+ def self.descriptor_errors(c)
192
+ errors = []
193
+
194
+ # validate: h is a hash
195
+ errors << "The command descriptor is not a hash" unless c.is_a?(Hash)
196
+
197
+ # validate: the hash c has a key :command
198
+ errors << "The command descriptor does not have a key :command" unless c.has_key?(:command)
199
+
200
+ # validate: the value of c[:command] is a string or symbol
201
+ errors << "The value of c[:command] is not a string and is not a symbol" unless c[:command].is_a?(String) || c[:command].is_a?(Symbol)
202
+
203
+ # validate: if the key :sudo exists, then its value is a boolean
204
+ if c.has_key?(:sudo)
205
+ errors << "The value of c[:sudo] is not a boolean" unless c[:sudo].is_a?(TrueClass) || c[:sudo].is_a?(FalseClass)
206
+ end
207
+
208
+ # if the parameter h[:name] is a symbol
209
+ if c[:command].is_a?(Symbol)
210
+ if c[:command] == :reboot
211
+ # :reboot is a reserved word, so it is fine to call :reboot
212
+ else
213
+ # validate: existis a routine with a the value c[:command].to_s on its :name key
214
+ errors << "The routine with the name #{c[:command].to_s} does not exist" unless BlackStack::Deployer::deployment_routines.select { |r| r.name == c[:command].to_s }.size > 0
215
+ end
216
+ end
217
+
218
+ # if c[:matches] exists
219
+ if c.has_key?(:matches)
220
+ # validate: the value of c[:matches] must by a regex or an array
221
+ errors << "The value of the key :matches is not a regex nor an array" unless c[:matches].is_a?(Regexp) || c[:matches].is_a?(Array)
222
+ # if c[:matches] is a array
223
+ if c[:matches].is_a?(Array)
224
+ # validate: each element in the the array c[:matches] is a regex
225
+ c[:matches].each do |m|
226
+ errors += BlackStack::Deployer::MatchModule.descriptor_errors(m)
227
+ end # each
228
+ end # if c[:matches].is_a?(Array)
229
+ end # if :matches exists
230
+
231
+ # if c[:nomatches] exists
232
+ if c.has_key?(:nomatches)
233
+ # validate: the value of c[:nomatches] must by a regex or an array
234
+ errors << "The value of the key :nomatches is not a regex nor an array" unless c[:nomatches].is_a?(Regexp) || c[:nomatches].is_a?(Array)
235
+ # if c[:nomatches] is a array
236
+ if c[:nomatches].is_a?(Array)
237
+ # validate: each element in the the array c[:nomatches] is a hash
238
+ c[:nomatches].each do |m|
239
+ errors += BlackStack::Deployer::NoMatchModule.descriptor_errors(m)
240
+ end # each
241
+ end # if c[:matches].is_a?(Array)
242
+ end # if :matches exists
243
+
244
+ # if c[:background] exists, it must be a boolean
245
+ if c.has_key?(:background)
246
+ errors << "The value of the key :background is not a boolean" unless c[:background].is_a?(TrueClass) || c[:background].is_a?(FalseClass)
247
+ end
248
+
249
+ #
250
+ errors.uniq
251
+ end # def self.descriptor_error(h)
252
+
253
+ def initialize(h)
254
+ errors = BlackStack::Deployer::CommandModule.descriptor_errors(h)
255
+ raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
256
+ self.command = h[:command]
257
+ self.sudo = h[:sudo].nil? ? true : h[:sudo]
258
+ self.matches = []
259
+ self.nomatches = []
260
+ if h.has_key?(:matches)
261
+ if h[:matches].is_a?(Regexp)
262
+ self.matches << BlackStack::Deployer::Match.new(h[:matches])
263
+ else
264
+ h[:matches].each do |m|
265
+ self.matches << BlackStack::Deployer::Match.new(m)
266
+ end
267
+ end
268
+ end
269
+ if h.has_key?(:nomatches)
270
+ if h[:nomatches].is_a?(Regexp)
271
+ self.nomatches << BlackStack::Deployer::NoMatch.new(h[:nomatches])
272
+ else
273
+ h[:nomatches].each do |m|
274
+ self.nomatches << BlackStack::Deployer::NoMatch.new(m)
275
+ end
276
+ end
277
+ end
278
+ if h.has_key?(:background)
279
+ self.background = h[:background]
280
+ else
281
+ self.background = false
282
+ end
283
+ end # def initialize(h)
284
+
285
+ def to_hash
286
+ h = {}
287
+ h[:command] = self.command
288
+ h[:sudo] = self.sudo
289
+ h[:matches] = []
290
+ h[:nomatches] = []
291
+ self.matches.each do |m|
292
+ h[:matches] << m.to_hash
293
+ end
294
+ self.nomatches.each do |m|
295
+ h[:nomatches] << m.to_hash
296
+ end
297
+ h
298
+ end # def to_hash
299
+
300
+ def run(node)
301
+ l = BlackStack::Deployer.logger
302
+ errors = []
303
+ code = self.command
304
+ output = nil
305
+
306
+ # if code is a symbol
307
+ if code.is_a?(Symbol)
308
+
309
+ # if code is equel than :reboot
310
+ if code == :reboot
311
+ # call the node reboot method
312
+ node.reboot
313
+ else
314
+ # look for a routine with this name
315
+ r = BlackStack::Deployer.routines.select { |r| r.name == code.to_s }.first
316
+ if !r.nil?
317
+ r.run(node)
318
+ else
319
+ raise "The routine #{code.to_s} does not exist"
320
+ end
321
+ end
322
+
323
+ # if code is a string
324
+ elsif code.is_a?(String)
325
+ # replacing parameters
326
+ code.scan(/%[a-zA-Z0-9\_]+%/).each do |p|
327
+ if p == '%eth0_ip%' # reserved parameter
328
+ # TODO: move the method eth0_ip to the blackstack-nodes library
329
+ code.gsub!(p, node.eth0_ip)
330
+ elsif p == '%timestamp%' # reserved parameter
331
+ # TODO: move this to a timestamp function on blackstack-core
332
+ code.gsub!(p, Time.now.to_s.gsub(/\D/, ''))
333
+ else
334
+ if node.parameters.has_key?(p.gsub(/%/, '').to_sym)
335
+ code.gsub!(p, node.parameters[p.gsub(/%/, '').to_sym].to_s)
336
+ else
337
+ raise "The parameter #{p} does not exist in the node descriptor #{node.parameters.to_s}"
338
+ end
339
+ end
340
+ end
341
+
342
+ # if the command is configured to run in background, and the flag show_ouput is off, then modify the code to run in background
343
+ if self.background && !BlackStack::Deployer.show_output
344
+ lines = code.strip.lines
345
+ total = lines.size
346
+ i = 0
347
+ lines.each { |l|
348
+ i += 1
349
+ if i == total
350
+ l.gsub!(/;$/, '> /dev/null 2>&1 &')
351
+ else
352
+ l.gsub!(/;$/, '> /dev/null 2>&1;')
353
+ end
354
+ }
355
+ code = lines.join("\n")
356
+ end
357
+
358
+ # running the command
359
+ l.logs "Show command output... " if BlackStack::Deployer.show_output
360
+ l.log "\n\nCommand:\n--------\n\n#{code} " if BlackStack::Deployer.show_output
361
+ output = node.exec(code, self.sudo)
362
+ l.log "\n\nOutput:\n-------\n\n#{output}" if BlackStack::Deployer.show_output
363
+ l.logf('done tracing.') if BlackStack::Deployer.show_output
364
+
365
+ # validation: at least one of the matches should happen
366
+ if self.matches.size > 0
367
+ i = 0
368
+ self.matches.each do |m|
369
+ if m.validate(output).size == 0 # no errors
370
+ i += 1
371
+ end
372
+ end
373
+ errors << "Command output doesn't match with any of the :matches" if i == 0
374
+ end # if self.matches.size > 0
375
+
376
+ # validation: no one of the nomatches should happen
377
+ self.nomatches.each do |m|
378
+ errors += m.validate(output)
379
+ end
380
+ end # elsif code.is_a?(String)
381
+
382
+ # return a hash descriptor of the command result
383
+ {
384
+ :command => self.command,
385
+ :code => code,
386
+ :output => output,
387
+ :errors => errors,
388
+ }
389
+ end # def run(node)
390
+ end # module CommandModule
391
+
392
+ # define attributes and methods of a command's match
393
+ module MatchModule
394
+ attr_accessor :match
395
+
396
+ def self.descriptor_errors(m)
397
+ errors = []
398
+ # validate the match is a regular expresion
399
+ errors << "The match is not a regex" unless m.is_a?(Regexp)
400
+ #
401
+ errors.uniq
402
+ end # def self.descriptor_error(h)
403
+
404
+ def initialize(h)
405
+ errors = BlackStack::Deployer::MatchModule.descriptor_errors(h)
406
+ raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
407
+ self.match = h
408
+ end
409
+
410
+ def to_hash
411
+ h = {}
412
+ h[:match] = self.match
413
+ h
414
+ end
415
+
416
+ def validate(output)
417
+ errors = []
418
+ errors << "The output of the command does not match the regular expression #{self.match.inspect}" unless output.match(self.match)
419
+ errors
420
+ end
421
+ end # module MatchModule
422
+
423
+ # define attributes and methods of a command's nomatch
424
+ module NoMatchModule
425
+ attr_accessor :nomatch, :error_description
426
+
427
+ def self.descriptor_errors(m)
428
+ errors = []
429
+ # validate the nomatch is a regular expresion
430
+ errors << "Each nomatch is not a regex nor a hash" unless m.is_a?(Hash) || m.is_a?(Regexp)
431
+ # if m is a hash
432
+ if m.is_a?(Hash)
433
+ # validate: the hash m has a key :nomatch
434
+ errors << "The hash descriptor of the nomatch does not have a key :nomatch" unless m.has_key?(:nomatch)
435
+ # validate: the value of m[:nomatch] is a string
436
+ errors << "The value of the key :nomatch is not a regex" unless m[:nomatch].is_a?(Regexp)
437
+ # validate: the hash m has a key :error_description
438
+ errors << "The hash descriptor of the nomatch does not have a key :error_description" unless m.has_key?(:error_description)
439
+ # validate: the value of m[:error_description] is a string
440
+ errors << "The value of the key :error_description is not a string" unless m[:error_description].is_a?(String)
441
+ end # if m.is_a?(Hash)
442
+ #
443
+ errors.uniq
444
+ end # def self.descriptor_error(h)
445
+
446
+ def initialize(h)
447
+ errors = BlackStack::Deployer::NoMatchModule.descriptor_errors(h)
448
+ raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
449
+ self.nomatch = h[:nomatch]
450
+ self.error_description = h[:error_description]
451
+ end
452
+
453
+ def to_hash
454
+ h = {}
455
+ h[:nomatch] = self.nomatch
456
+ h[:error_description] = self.error_description
457
+ h
458
+ end
459
+
460
+ def validate(output)
461
+ errors = []
462
+ if !self.error_description.nil?
463
+ errors << self.error_description if output.match(self.nomatch)
464
+ else
465
+ errors << "The output of the command matches the regular expression #{self.nomatch.inspect}" if output.match(self.nomatch)
466
+ end
467
+ errors
468
+ end
469
+
470
+ end # module NoMatchModule
471
+
472
+
473
+ # TODO: declare these classes (stub and skeleton) using blackstack-rpc
474
+ #
475
+ # Stub Classes
476
+ # These classes represents a node, without using connection to the database.
477
+ # Use this class at the client side.
478
+ class Node
479
+ include BlackStack::Deployer::NodeModule
480
+ end # class Node
481
+
482
+ class Command
483
+ include BlackStack::Deployer::CommandModule
484
+ end # class Command
485
+
486
+ class Routine
487
+ include BlackStack::Deployer::RoutineModule
488
+ end # class Routine
489
+
490
+ class Match
491
+ include BlackStack::Deployer::MatchModule
492
+ end # class Match
493
+
494
+ class NoMatch
495
+ include BlackStack::Deployer::NoMatchModule
496
+ end # class NoMatch
497
+
498
+ # add a node to the list of nodes.
499
+ def self.add_node(h)
500
+ errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
501
+ raise errors.join(".\n") unless errors.empty?
502
+ @@nodes << BlackStack::Deployer::Node.new(h, @@logger)
503
+ end # def
504
+
505
+ # add an array of nodes to the list of nodes.
506
+ def self.add_nodes(a)
507
+ # validate: the parameter a is an array
508
+ raise "The parameter a is not an array" unless a.is_a?(Array)
509
+ a.each { |h| BlackStack::Deployer.add_node(h) }
510
+ end # def
511
+
512
+ # remove all exisiting nodes in he list of nodes.
513
+ # then, add the nodes in the parameter a to the list of nodes.
514
+ def self.set_nodes(a)
515
+ @@nodes.clear
516
+ BlackStack::Deployer.add_nodes(a)
517
+ end # def
518
+
519
+ # add a routine to the list of routines.
520
+ def self.add_routine(h)
521
+ errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
522
+ raise errors.join(".\n") unless errors.empty?
523
+ @@routines << BlackStack::Deployer::Routine.new(h)
524
+ end # def
525
+
526
+ # add an array of routines to the list of routines.
527
+ def self.add_routines(a)
528
+ # validate: the parameter a is an array
529
+ raise "The parameter a is not an array" unless a.is_a?(Array)
530
+ a.each { |h| BlackStack::Deployer.add_routine(h) }
531
+ end # def
532
+
533
+ # remove all exisiting routines in he list of routines.
534
+ # then, add the routines in the parameter a to the list of routines.
535
+ def self.set_routines(a)
536
+ @@routines.clear
537
+ BlackStack::Deployer.add_routines(a)
538
+ end # def
539
+
540
+ # running a routine on a node
541
+ def self.run_routine(node_name, routine_name)
542
+ errors = []
543
+
544
+ # find the node with the value node_name in the key :name
545
+ n = @@nodes.select { |n| n.name == node_name }.first
546
+
547
+ # find the routine with the value routine_name in the key :name
548
+ r = @@routines.select { |r| r.name == routine_name }.first
549
+
550
+ # validate: the node n exists
551
+ errors << "Node #{node_name} not found" unless n
552
+
553
+ # validate: the routine r exists
554
+ errors << "Routine #{routine_name} not found" unless r
555
+
556
+ # raise exception if any error has been found
557
+ raise "The routine #{routine_name} cannot be run on the node #{node_name}: #{errors.uniq.join(".\n")}" if errors.length > 0
558
+
559
+ # connect the node
560
+ #self.logger.logs "Connecting to node #{n.name}... "
561
+ n.connect
562
+ #self.logger.done
563
+
564
+ # run the routine
565
+ #self.logger.logs "Running routine #{r.name}... "
566
+ r.run(n)
567
+ #self.logger.done
568
+
569
+ # disconnect the node
570
+ #self.logger.logs "Disconnecting from node #{n.name}... "
571
+ n.disconnect
572
+ #self.logger.done
573
+
574
+ end # def
575
+
576
+ module DB
577
+ LOCKFILE = './blackstack-deployer.lock'
578
+ @@checkpoint = nil
579
+ @@superhuser = nil
580
+ @@ndb = nil
581
+ @@folder = nil
582
+
583
+ def self.set_checkpoint(s)
584
+ @@checkpoint = s
585
+ end
586
+
587
+ def self.checkpoint
588
+ @@checkpoint
589
+ end
590
+
591
+ def self.save_checkpoint(lockfilename=BlackStack::Deployer::DB::LOCKFILE)
592
+ File.new(lockfilename, "w").write(@@checkpoint)
593
+ end
594
+
595
+ def self.load_checkpoint(lockfilename=BlackStack::Deployer::DB::LOCKFILE)
596
+ if File.exists?(lockfilename)
597
+ @@checkpoint = File.new(lockfilename, "r").read
598
+ else
599
+ @@checkpoint = nil
600
+ end
601
+ @@checkpoint
602
+ end
603
+
604
+ def self.connect(s)
605
+ @@db = Sequel::connect(s)
606
+ end # def
607
+
608
+ def self.set_folder(s)
609
+ @@folder = s
610
+ end # def
611
+
612
+
613
+ # Return `true` if the name of the file matches with `/\.transactions\./`, and it doesn't match with `/\.sentences\./`, and the matches with `/\.transactions\./` are no more than one.
614
+ # Otherwise, return `false`.
615
+ # This method should not be called directly by user code.
616
+ def self.is_transactions_file?(filename)
617
+ filename =~ /\.transactions\./ && filename !~ /\.sentences\./ && filename.scan(/\.transactions\./).size == 1
618
+ end
619
+
620
+ # Return `true` if the name of the file matches with `/\.sentences\./`, and it doesn't match with `/\.transactions\./`, and the matches with `/\.sentences\./` are no more than one.
621
+ # Otherwise({, return `false`.
622
+ # This method should not be called directly by user code.
623
+ def self.is_sentences_file?(filename)
624
+ filename =~ /\.sentences\./ && filename !~ /\.transactions\./ && filename.scan(/\.sentences\./).size == 1
625
+ end
626
+
627
+ # Method to process an `.sql` file with transactions code, separated by `BEGIN;` and `COMMIT;` statements.
628
+ # Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
629
+ # This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.tsql\./`.
630
+ # This method should not be called directly by user code.
631
+ def self.execute_transactions(sql)
632
+ # TODO: Code Me!
633
+ end # def db_execute_tsql
634
+
635
+ # Method to process an `.sql` file with one sql sentence by line.
636
+ # Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
637
+ # This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.sentences\./`.
638
+ # This method should not be called directly by user code.
639
+ def self.execute_sentences(sql, chunk_size=200)
640
+ tlogger = BlackStack::Deployer::logger
641
+
642
+ # Fix issue: Ruby `split': invalid byte sequence in UTF-8 (ArgumentError)
643
+ # Reference: https://stackoverflow.com/questions/11065962/ruby-split-invalid-byte-sequence-in-utf-8-argumenterror
644
+ #
645
+ # Fix issue: `PG::SyntaxError: ERROR: at or near "��truncate": syntax error`
646
+ #
647
+ sql.encode!('UTF-8', :invalid => :replace, :replace => '')
648
+
649
+ # Remove null bytes to avoid error: `String contains null byte`
650
+ # Reference: https://stackoverflow.com/questions/29320369/coping-with-string-contains-null-byte-sent-from-users
651
+ sql.gsub!("\u0000", "")
652
+
653
+ # Get the array of sentences
654
+ tlogger.logs "Splitting the sql sentences... "
655
+ sentences = sql.split(/;/i)
656
+ tlogger.logf "done (#{sentences.size})"
657
+
658
+ # Chunk the array into parts of chunk_size elements
659
+ # Reference: https://stackoverflow.com/questions/2699584/how-to-split-chunk-a-ruby-array-into-parts-of-x-elements
660
+ tlogger.logs "Bunlding the array of sentences into chunks of #{chunk_size} each... "
661
+ chunks = sentences.each_slice(chunk_size).to_a
662
+ tlogger.logf "done (#{chunks.size})"
663
+
664
+ chunk_number = -1
665
+ chunks.each { |chunk|
666
+ chunk_number += 1
667
+ statement = chunk.join(";\n").to_s.strip
668
+ tlogger.logs "lines #{chunk_size*chunk_number+1} to #{chunk_size*chunk_number+chunk.size} of #{sentences.size}... "
669
+ begin
670
+ @@db.execute(statement) #if statement.to_s.strip.size > 0
671
+ tlogger.done
672
+ rescue => e
673
+ tlogger.logf e.to_s
674
+ raise "Error executing statement: #{statement}\n#{e.message}"
675
+ end
676
+ }
677
+ tlogger.done
678
+ end # def db_execute_sql_sentences_file
679
+
680
+ # Run a series of `.sql` files with updates to the database.
681
+ #
682
+ def self.deploy(save_checkpoints=false, lockfilename=BlackStack::Deployer::DB::LOCKFILE)
683
+ tlogger = BlackStack::Deployer::logger
684
+ # get list of `.sql` files in the directory `sql_path`, with a name higher than `last_filename`, sorted by name.
685
+ Dir.entries(@@folder).select {
686
+ |filename| filename =~ /\.sql$/ && filename > @@checkpoint.to_s
687
+ }.uniq.sort.each { |filename|
688
+ fullfilename = "#{@@folder}/#{filename}"
689
+
690
+ tlogger.logs "#{fullfilename}... "
691
+ BlackStack::Deployer::DB::execute_sentences( File.open(fullfilename).read )
692
+ tlogger.done
693
+
694
+ tlogger.logs "Updating checkpoint... "
695
+ BlackStack::Deployer::DB::set_checkpoint filename
696
+ tlogger.done
697
+
698
+ tlogger.logs 'Saving checkpoint... '
699
+ if save_checkpoints
700
+ BlackStack::Deployer::DB::save_checkpoint(lockfilename)
701
+ tlogger.done
702
+ else
703
+ tlogger.logf 'disabled'
704
+ end
705
+ }
706
+ end # def
707
+ end # module DB
708
+
709
+ # deploying all db-updates and run all routines on all nodes
710
+ def self.deploy()
711
+ tlogger = BlackStack::Deployer::logger
712
+
713
+ @@nodes.each { |n|
714
+ tlogger.logs "Node #{n.name}... "
715
+ n.deploy()
716
+ tlogger.done
717
+ }
718
+ end # def
719
+
720
+ end # module Deployer
721
+
670
722
  end # module BlackStack