blackstack-deployer 1.2.10 → 1.2.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/blackstack-deployer.rb +678 -669
- metadata +9 -9
data/lib/blackstack-deployer.rb
CHANGED
@@ -1,670 +1,679 @@
|
|
1
|
-
require 'blackstack-nodes'
|
2
|
-
require 'sequel'
|
3
|
-
|
4
|
-
module BlackStack
|
5
|
-
# Deployer is a library that can be used to deploy a cluster of nodes.
|
6
|
-
module Deployer
|
7
|
-
@@logger = BlackStack::BaseLogger.new(nil)
|
8
|
-
@@nodes = []
|
9
|
-
@@routines = []
|
10
|
-
|
11
|
-
# set the logger
|
12
|
-
def self.set_logger(i_logger)
|
13
|
-
@@logger = i_logger
|
14
|
-
end
|
15
|
-
|
16
|
-
# get the logger assigned to the module
|
17
|
-
def self.logger
|
18
|
-
@@logger
|
19
|
-
end # def self.errors
|
20
|
-
|
21
|
-
# get the array of nodes assigned to the module
|
22
|
-
def self.nodes
|
23
|
-
@@nodes
|
24
|
-
end # def self.nodes
|
25
|
-
|
26
|
-
# get the array of routines assigned to the module
|
27
|
-
def self.routines
|
28
|
-
@@routines
|
29
|
-
end # def self.routines
|
30
|
-
|
31
|
-
# inherit BlackStack::Infrastructure::NodeModule, including features of deployer.
|
32
|
-
module NodeModule
|
33
|
-
attr_accessor :deployment_routine, :parameters
|
34
|
-
|
35
|
-
include BlackStack::Infrastructure::NodeModule
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
a
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
end
|
60
|
-
|
61
|
-
def
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
h
|
71
|
-
h[:deployment_routine]
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
h[:
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
errors.
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
h[:commands]
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
h
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
h
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
ret
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
if
|
193
|
-
|
194
|
-
errors << "The value of
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
h
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
#
|
287
|
-
code
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
#
|
305
|
-
#
|
306
|
-
|
307
|
-
#
|
308
|
-
#
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
:
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
errors
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
def
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
class
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
#
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
#
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
#
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
#
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
#
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
#
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
errors
|
500
|
-
|
501
|
-
#
|
502
|
-
|
503
|
-
|
504
|
-
#
|
505
|
-
|
506
|
-
|
507
|
-
#
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
@@checkpoint
|
550
|
-
end
|
551
|
-
|
552
|
-
def self.
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
#
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
#
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
#
|
578
|
-
#
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
#
|
585
|
-
#
|
586
|
-
# This method
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
#
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
#
|
607
|
-
# Reference: https://stackoverflow.com/questions/
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
1
|
+
require 'blackstack-nodes'
|
2
|
+
require 'sequel'
|
3
|
+
|
4
|
+
module BlackStack
|
5
|
+
# Deployer is a library that can be used to deploy a cluster of nodes.
|
6
|
+
module Deployer
|
7
|
+
@@logger = BlackStack::BaseLogger.new(nil)
|
8
|
+
@@nodes = []
|
9
|
+
@@routines = []
|
10
|
+
|
11
|
+
# set the logger
|
12
|
+
def self.set_logger(i_logger)
|
13
|
+
@@logger = i_logger
|
14
|
+
end
|
15
|
+
|
16
|
+
# get the logger assigned to the module
|
17
|
+
def self.logger
|
18
|
+
@@logger
|
19
|
+
end # def self.errors
|
20
|
+
|
21
|
+
# get the array of nodes assigned to the module
|
22
|
+
def self.nodes
|
23
|
+
@@nodes
|
24
|
+
end # def self.nodes
|
25
|
+
|
26
|
+
# get the array of routines assigned to the module
|
27
|
+
def self.routines
|
28
|
+
@@routines
|
29
|
+
end # def self.routines
|
30
|
+
|
31
|
+
# inherit BlackStack::Infrastructure::NodeModule, including features of deployer.
|
32
|
+
module NodeModule
|
33
|
+
attr_accessor :deployment_routine, :parameters
|
34
|
+
|
35
|
+
include BlackStack::Infrastructure::NodeModule
|
36
|
+
|
37
|
+
def self.eth0_ip(insterface)
|
38
|
+
ret = nil
|
39
|
+
a = `ip addr show dev #{insterface}`.scan(/inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
|
40
|
+
if a.size > 0
|
41
|
+
ret = a.last.to_s.gsub(/inet /, '')
|
42
|
+
else
|
43
|
+
raise "Cannot get ip address of the interface #{insterface}"
|
44
|
+
end
|
45
|
+
ret
|
46
|
+
end
|
47
|
+
|
48
|
+
# get the IP address for an interface using the ip addr command.
|
49
|
+
# this is a helper method for installing cockroachdb nodes.
|
50
|
+
def eth0_ip()
|
51
|
+
ret = nil
|
52
|
+
a = self.ssh.exec!("ip addr show dev #{parameters[:laninterface]}").scan(/inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/)
|
53
|
+
if a.size > 0
|
54
|
+
ret = a.last.to_s.gsub(/inet /, '')
|
55
|
+
else
|
56
|
+
raise "Cannot get ip address of the interface #{parameters[:laninterface]}"
|
57
|
+
end
|
58
|
+
ret
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.descriptor_errors(h)
|
62
|
+
errors = BlackStack::Infrastructure::NodeModule.descriptor_errors(h)
|
63
|
+
|
64
|
+
# validate: does not exist any other element in @@nodes with the same value for the parameter h[:name]
|
65
|
+
errors << "The parameter h[:name] is not unique" if BlackStack::Deployer.nodes.select{|n| n.name == h[:name]}.length > 0
|
66
|
+
|
67
|
+
# validate: h[:deployment_routine] is not nil
|
68
|
+
errors << "The parameter h[:deployment_routine] is required" if h[:deployment_routine].nil?
|
69
|
+
|
70
|
+
# validate: h[:deployment_routine] is a string
|
71
|
+
errors << "The parameter h[:deployment_routine] is not a string" unless h[:deployment_routine].is_a?(String)
|
72
|
+
|
73
|
+
# return list of errors
|
74
|
+
errors.uniq
|
75
|
+
end
|
76
|
+
|
77
|
+
def initialize(h, i_logger=nil)
|
78
|
+
self.parameters = h
|
79
|
+
errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
|
80
|
+
raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
|
81
|
+
super(h, i_logger)
|
82
|
+
self.deployment_routine = h[:deployment_routine]
|
83
|
+
end # def self.create(h)
|
84
|
+
|
85
|
+
def to_hash
|
86
|
+
h = super
|
87
|
+
h[:deployment_routine] = self.deployment_routine
|
88
|
+
h
|
89
|
+
end # def to_hash
|
90
|
+
|
91
|
+
def deploy()
|
92
|
+
BlackStack::Deployer::run_routine(self.name, self.deployment_routine);
|
93
|
+
end
|
94
|
+
end # module NodeModule
|
95
|
+
|
96
|
+
# define attributes and methods of a deployer routine
|
97
|
+
module RoutineModule
|
98
|
+
attr_accessor :name, :commands
|
99
|
+
|
100
|
+
def self.descriptor_errors(h)
|
101
|
+
errors = []
|
102
|
+
|
103
|
+
# validate: the parameter h is a hash
|
104
|
+
errors << "The parameter h is not a hash" unless h.is_a?(Hash)
|
105
|
+
|
106
|
+
# validate: the paramerer h has a key :name
|
107
|
+
errors << "The parameter h does not have a key :name" unless h.has_key?(:name)
|
108
|
+
|
109
|
+
# validate: the paramerer h has a key :command
|
110
|
+
errors << "The parameter h does not have a key :commands" unless h.has_key?(:commands)
|
111
|
+
|
112
|
+
# validate: the parameter h[:name] is a string or a symbol
|
113
|
+
errors << "The parameter h[:name] is not a string" unless h[:name].is_a?(String)
|
114
|
+
|
115
|
+
# validate: the parameter h[:name] is not 'reboot' because it is a reserved name
|
116
|
+
errors << "The parameter h[:name] is a reserved name (#{h[:name].to_s})" if h[:name] == 'reboot'
|
117
|
+
|
118
|
+
# validate: the parameter h[:commands] is required
|
119
|
+
errors << "The parameter h[:commands] is required" if h[:commands].nil?
|
120
|
+
|
121
|
+
# validate: the parametrer h[:commands] is an array
|
122
|
+
errors << "The parameter h[:commands] is not an array" unless h[:commands].is_a?(Array)
|
123
|
+
|
124
|
+
# validate: the parameter h[:commands] has at least one element
|
125
|
+
errors << "The parameter h[:commands] does not have at least one element" unless h[:commands].size > 0
|
126
|
+
|
127
|
+
# validate: each element of the array h[:commands] is a hash
|
128
|
+
h[:commands].each do |c|
|
129
|
+
errors += BlackStack::Deployer::CommandModule.descriptor_errors(c)
|
130
|
+
end # h[:commands].each do |c|
|
131
|
+
|
132
|
+
errors.uniq
|
133
|
+
end # def self.descriptor_error(h)
|
134
|
+
|
135
|
+
def initialize(h)
|
136
|
+
errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
|
137
|
+
raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
|
138
|
+
self.name = h[:name]
|
139
|
+
self.commands = []
|
140
|
+
h[:commands].each do |c|
|
141
|
+
self.commands << BlackStack::Deployer::Command.new(c)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def to_hash
|
146
|
+
h = {}
|
147
|
+
h[:name] = self.name
|
148
|
+
h[:commands] = []
|
149
|
+
self.commands.each do |c|
|
150
|
+
h[:commands] << c.to_hash
|
151
|
+
end
|
152
|
+
h
|
153
|
+
end
|
154
|
+
|
155
|
+
def run(node)
|
156
|
+
ret = []
|
157
|
+
self.commands.each do |c|
|
158
|
+
BlackStack::Deployer.logger.logs "Running command: #{c.command.to_s}... "
|
159
|
+
h = c.run(node)
|
160
|
+
ret << h
|
161
|
+
|
162
|
+
#BlackStack::Deployer.logger.logs "Result: "
|
163
|
+
#BlackStack::Deployer.logger.logf h.to_s
|
164
|
+
|
165
|
+
if h[:errors].size == 0
|
166
|
+
BlackStack::Deployer.logger.done
|
167
|
+
else
|
168
|
+
BlackStack::Deployer.logger.logf('error: ' + h.to_s)
|
169
|
+
raise "Error running command: #{h.to_s}"
|
170
|
+
end
|
171
|
+
end
|
172
|
+
ret
|
173
|
+
end # def run(node)
|
174
|
+
end # module RoutineModule
|
175
|
+
|
176
|
+
# define attributes and methods of a routine's command
|
177
|
+
module CommandModule
|
178
|
+
attr_accessor :command, :matches, :nomatches, :sudo
|
179
|
+
|
180
|
+
def self.descriptor_errors(c)
|
181
|
+
errors = []
|
182
|
+
|
183
|
+
# validate: h is a hash
|
184
|
+
errors << "The command descriptor is not a hash" unless c.is_a?(Hash)
|
185
|
+
|
186
|
+
# validate: the hash c has a key :command
|
187
|
+
errors << "The command descriptor does not have a key :command" unless c.has_key?(:command)
|
188
|
+
|
189
|
+
# validate: the value of c[:command] is a string or symbol
|
190
|
+
errors << "The value of c[:command] is not a string and is not a symbol" unless c[:command].is_a?(String) || c[:command].is_a?(Symbol)
|
191
|
+
|
192
|
+
# validate: if the key :sudo exists, then its value is a boolean
|
193
|
+
if c.has_key?(:sudo)
|
194
|
+
errors << "The value of c[:sudo] is not a boolean" unless c[:sudo].is_a?(TrueClass) || c[:sudo].is_a?(FalseClass)
|
195
|
+
end
|
196
|
+
|
197
|
+
# if the parameter h[:name] is a symbol
|
198
|
+
if c[:command].is_a?(Symbol)
|
199
|
+
if c[:command] == :reboot
|
200
|
+
# :reboot is a reserved word, so it is fine to call :reboot
|
201
|
+
else
|
202
|
+
# validate: existis a routine with a the value c[:command].to_s on its :name key
|
203
|
+
errors << "The routine with the name #{c[:command].to_s} does not exist" unless BlackStack::Deployer::routines.select { |r| r.name == c[:command].to_s }.size > 0
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
# if c[:matches] exists
|
208
|
+
if c.has_key?(:matches)
|
209
|
+
# validate: the value of c[:matches] must by a regex or an array
|
210
|
+
errors << "The value of the key :matches is not a regex nor an array" unless c[:matches].is_a?(Regexp) || c[:matches].is_a?(Array)
|
211
|
+
# if c[:matches] is a array
|
212
|
+
if c[:matches].is_a?(Array)
|
213
|
+
# validate: each element in the the array c[:matches] is a regex
|
214
|
+
c[:matches].each do |m|
|
215
|
+
errors += BlackStack::Deployer::MatchModule.descriptor_errors(m)
|
216
|
+
end # each
|
217
|
+
end # if c[:matches].is_a?(Array)
|
218
|
+
end # if :matches exists
|
219
|
+
|
220
|
+
# if c[:nomatches] exists
|
221
|
+
if c.has_key?(:nomatches)
|
222
|
+
# validate: the value of c[:nomatches] must by a regex or an array
|
223
|
+
errors << "The value of the key :nomatches is not a regex nor an array" unless c[:nomatches].is_a?(Regexp) || c[:nomatches].is_a?(Array)
|
224
|
+
# if c[:nomatches] is a array
|
225
|
+
if c[:nomatches].is_a?(Array)
|
226
|
+
# validate: each element in the the array c[:nomatches] is a hash
|
227
|
+
c[:nomatches].each do |m|
|
228
|
+
errors += BlackStack::Deployer::NoMatchModule.descriptor_errors(m)
|
229
|
+
end # each
|
230
|
+
end # if c[:matches].is_a?(Array)
|
231
|
+
end # if :matches exists
|
232
|
+
#
|
233
|
+
errors.uniq
|
234
|
+
end # def self.descriptor_error(h)
|
235
|
+
|
236
|
+
def initialize(h)
|
237
|
+
errors = BlackStack::Deployer::CommandModule.descriptor_errors(h)
|
238
|
+
raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
|
239
|
+
self.command = h[:command]
|
240
|
+
self.sudo = h[:sudo].nil? ? true : h[:sudo]
|
241
|
+
self.matches = []
|
242
|
+
self.nomatches = []
|
243
|
+
if h.has_key?(:matches)
|
244
|
+
if h[:matches].is_a?(Regexp)
|
245
|
+
self.matches << BlackStack::Deployer::Match.new(h[:matches])
|
246
|
+
else
|
247
|
+
h[:matches].each do |m|
|
248
|
+
self.matches << BlackStack::Deployer::Match.new(m)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
if h.has_key?(:nomatches)
|
253
|
+
if h[:nomatches].is_a?(Regexp)
|
254
|
+
self.nomatches << BlackStack::Deployer::NoMatch.new(h[:nomatches])
|
255
|
+
else
|
256
|
+
h[:nomatches].each do |m|
|
257
|
+
self.nomatches << BlackStack::Deployer::NoMatch.new(m)
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|
261
|
+
end # def initialize(h)
|
262
|
+
|
263
|
+
def to_hash
|
264
|
+
h = {}
|
265
|
+
h[:command] = self.command
|
266
|
+
h[:sudo] = self.sudo
|
267
|
+
h[:matches] = []
|
268
|
+
h[:nomatches] = []
|
269
|
+
self.matches.each do |m|
|
270
|
+
h[:matches] << m.to_hash
|
271
|
+
end
|
272
|
+
self.nomatches.each do |m|
|
273
|
+
h[:nomatches] << m.to_hash
|
274
|
+
end
|
275
|
+
h
|
276
|
+
end # def to_hash
|
277
|
+
|
278
|
+
def run(node)
|
279
|
+
errors = []
|
280
|
+
code = self.command
|
281
|
+
output = nil
|
282
|
+
|
283
|
+
# if code is a symbol
|
284
|
+
if code.is_a?(Symbol)
|
285
|
+
|
286
|
+
# if code is equel than :reboot
|
287
|
+
if code == :reboot
|
288
|
+
# call the node reboot method
|
289
|
+
node.reboot
|
290
|
+
else
|
291
|
+
# look for a routine with this name
|
292
|
+
r = BlackStack::Deployer.routines.select { |r| r.name == code.to_s }.first
|
293
|
+
if !r.nil?
|
294
|
+
r.run(node)
|
295
|
+
else
|
296
|
+
raise "The routine #{code.to_s} does not exist"
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
# if code is a string
|
301
|
+
elsif code.is_a?(String)
|
302
|
+
# replacing parameters
|
303
|
+
code.scan(/%[a-zA-Z0-9\_]+%/).each do |p|
|
304
|
+
if p == '%eth0_ip%' # reserved parameter
|
305
|
+
# TODO: move the method eth0_ip to the blackstack-nodes library
|
306
|
+
code.gsub!(p, node.eth0_ip)
|
307
|
+
elsif p == '%timestamp%' # reserved parameter
|
308
|
+
# TODO: move this to a timestamp function on blackstack-core
|
309
|
+
code.gsub!(p, Time.now.to_s.gsub(/\D/, ''))
|
310
|
+
else
|
311
|
+
if node.parameters.has_key?(p.gsub(/%/, '').to_sym)
|
312
|
+
code.gsub!(p, node.parameters[p.gsub(/%/, '').to_sym].to_s)
|
313
|
+
else
|
314
|
+
raise "The parameter #{p} does not exist in the node descriptor #{node.parameters.to_s}"
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
|
319
|
+
# running the command
|
320
|
+
output = node.exec(code, self.sudo)
|
321
|
+
|
322
|
+
# validation: at least one of the matches should happen
|
323
|
+
if self.matches.size > 0
|
324
|
+
i = 0
|
325
|
+
self.matches.each do |m|
|
326
|
+
if m.validate(output).size == 0 # no errors
|
327
|
+
i += 1
|
328
|
+
end
|
329
|
+
end
|
330
|
+
errors << "Command output doesn't match with any of the :matches" if i == 0
|
331
|
+
end # if self.matches.size > 0
|
332
|
+
|
333
|
+
# validation: no one of the nomatches should happen
|
334
|
+
self.nomatches.each do |m|
|
335
|
+
errors += m.validate(output)
|
336
|
+
end
|
337
|
+
end # elsif code.is_a?(String)
|
338
|
+
|
339
|
+
# return a hash descriptor of the command result
|
340
|
+
{
|
341
|
+
:command => self.command,
|
342
|
+
:code => code,
|
343
|
+
:output => output,
|
344
|
+
:errors => errors,
|
345
|
+
}
|
346
|
+
end # def run(node)
|
347
|
+
end # module CommandModule
|
348
|
+
|
349
|
+
# define attributes and methods of a command's match
|
350
|
+
module MatchModule
|
351
|
+
attr_accessor :match
|
352
|
+
|
353
|
+
def self.descriptor_errors(m)
|
354
|
+
errors = []
|
355
|
+
# validate the match is a regular expresion
|
356
|
+
errors << "The match is not a regex" unless m.is_a?(Regexp)
|
357
|
+
#
|
358
|
+
errors.uniq
|
359
|
+
end # def self.descriptor_error(h)
|
360
|
+
|
361
|
+
def initialize(h)
|
362
|
+
errors = BlackStack::Deployer::MatchModule.descriptor_errors(h)
|
363
|
+
raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
|
364
|
+
self.match = h
|
365
|
+
end
|
366
|
+
|
367
|
+
def to_hash
|
368
|
+
h = {}
|
369
|
+
h[:match] = self.match
|
370
|
+
h
|
371
|
+
end
|
372
|
+
|
373
|
+
def validate(output)
|
374
|
+
errors = []
|
375
|
+
errors << "The output of the command does not match the regular expression #{self.match.inspect}" unless output.match(self.match)
|
376
|
+
errors
|
377
|
+
end
|
378
|
+
end # module MatchModule
|
379
|
+
|
380
|
+
# define attributes and methods of a command's nomatch
|
381
|
+
module NoMatchModule
|
382
|
+
attr_accessor :nomatch, :error_description
|
383
|
+
|
384
|
+
def self.descriptor_errors(m)
|
385
|
+
errors = []
|
386
|
+
# validate the nomatch is a regular expresion
|
387
|
+
errors << "Each nomatch is not a regex nor a hash" unless m.is_a?(Hash) || m.is_a?(Regexp)
|
388
|
+
# if m is a hash
|
389
|
+
if m.is_a?(Hash)
|
390
|
+
# validate: the hash m has a key :nomatch
|
391
|
+
errors << "The hash descriptor of the nomatch does not have a key :nomatch" unless m.has_key?(:nomatch)
|
392
|
+
# validate: the value of m[:nomatch] is a string
|
393
|
+
errors << "The value of the key :nomatch is not a regex" unless m[:nomatch].is_a?(Regexp)
|
394
|
+
# validate: the hash m has a key :error_description
|
395
|
+
errors << "The hash descriptor of the nomatch does not have a key :error_description" unless m.has_key?(:error_description)
|
396
|
+
# validate: the value of m[:error_description] is a string
|
397
|
+
errors << "The value of the key :error_description is not a string" unless m[:error_description].is_a?(String)
|
398
|
+
end # if m.is_a?(Hash)
|
399
|
+
#
|
400
|
+
errors.uniq
|
401
|
+
end # def self.descriptor_error(h)
|
402
|
+
|
403
|
+
def initialize(h)
|
404
|
+
errors = BlackStack::Deployer::NoMatchModule.descriptor_errors(h)
|
405
|
+
raise "The node descriptor is not valid: #{errors.uniq.join(".\n")}" if errors.length > 0
|
406
|
+
self.nomatch = h[:nomatch]
|
407
|
+
self.error_description = h[:error_description]
|
408
|
+
end
|
409
|
+
|
410
|
+
def to_hash
|
411
|
+
h = {}
|
412
|
+
h[:nomatch] = self.nomatch
|
413
|
+
h[:error_description] = self.error_description
|
414
|
+
h
|
415
|
+
end
|
416
|
+
|
417
|
+
def validate(output)
|
418
|
+
errors = []
|
419
|
+
if !self.error_description.nil?
|
420
|
+
errors << self.error_description if output.match(self.nomatch)
|
421
|
+
else
|
422
|
+
errors << "The output of the command matches the regular expression #{self.nomatch.inspect}" if output.match(self.nomatch)
|
423
|
+
end
|
424
|
+
errors
|
425
|
+
end
|
426
|
+
|
427
|
+
end # module NoMatchModule
|
428
|
+
|
429
|
+
|
430
|
+
# TODO: declare these classes (stub and skeleton) using blackstack-rpc
|
431
|
+
#
|
432
|
+
# Stub Classes
|
433
|
+
# These classes represents a node, without using connection to the database.
|
434
|
+
# Use this class at the client side.
|
435
|
+
class Node
|
436
|
+
include BlackStack::Deployer::NodeModule
|
437
|
+
end # class Node
|
438
|
+
|
439
|
+
class Command
|
440
|
+
include BlackStack::Deployer::CommandModule
|
441
|
+
end # class Command
|
442
|
+
|
443
|
+
class Routine
|
444
|
+
include BlackStack::Deployer::RoutineModule
|
445
|
+
end # class Routine
|
446
|
+
|
447
|
+
class Match
|
448
|
+
include BlackStack::Deployer::MatchModule
|
449
|
+
end # class Match
|
450
|
+
|
451
|
+
class NoMatch
|
452
|
+
include BlackStack::Deployer::NoMatchModule
|
453
|
+
end # class NoMatch
|
454
|
+
|
455
|
+
# add a node to the list of nodes.
|
456
|
+
def self.add_node(h)
|
457
|
+
errors = BlackStack::Deployer::NodeModule.descriptor_errors(h)
|
458
|
+
raise errors.join(".\n") unless errors.empty?
|
459
|
+
@@nodes << BlackStack::Deployer::Node.new(h, @@logger)
|
460
|
+
end # def
|
461
|
+
|
462
|
+
# add an array of nodes to the list of nodes.
|
463
|
+
def self.add_nodes(a)
|
464
|
+
# validate: the parameter a is an array
|
465
|
+
raise "The parameter a is not an array" unless a.is_a?(Array)
|
466
|
+
a.each { |h| BlackStack::Deployer.add_node(h) }
|
467
|
+
end # def
|
468
|
+
|
469
|
+
# remove all exisiting nodes in he list of nodes.
|
470
|
+
# then, add the nodes in the parameter a to the list of nodes.
|
471
|
+
def self.set_nodes(a)
|
472
|
+
@@nodes.clear
|
473
|
+
BlackStack::Deployer.add_nodes(a)
|
474
|
+
end # def
|
475
|
+
|
476
|
+
# add a routine to the list of routines.
|
477
|
+
def self.add_routine(h)
|
478
|
+
errors = BlackStack::Deployer::RoutineModule.descriptor_errors(h)
|
479
|
+
raise errors.join(".\n") unless errors.empty?
|
480
|
+
@@routines << BlackStack::Deployer::Routine.new(h)
|
481
|
+
end # def
|
482
|
+
|
483
|
+
# add an array of routines to the list of routines.
|
484
|
+
def self.add_routines(a)
|
485
|
+
# validate: the parameter a is an array
|
486
|
+
raise "The parameter a is not an array" unless a.is_a?(Array)
|
487
|
+
a.each { |h| BlackStack::Deployer.add_routine(h) }
|
488
|
+
end # def
|
489
|
+
|
490
|
+
# remove all exisiting routines in he list of routines.
|
491
|
+
# then, add the routines in the parameter a to the list of routines.
|
492
|
+
def self.set_routines(a)
|
493
|
+
@@routines.clear
|
494
|
+
BlackStack::Deployer.add_routines(a)
|
495
|
+
end # def
|
496
|
+
|
497
|
+
# running a routine on a node
|
498
|
+
def self.run_routine(node_name, routine_name)
|
499
|
+
errors = []
|
500
|
+
|
501
|
+
# find the node with the value node_name in the key :name
|
502
|
+
n = @@nodes.select { |n| n.name == node_name }.first
|
503
|
+
|
504
|
+
# find the routine with the value routine_name in the key :name
|
505
|
+
r = @@routines.select { |r| r.name == routine_name }.first
|
506
|
+
|
507
|
+
# validate: the node n exists
|
508
|
+
errors << "Node #{node_name} not found" unless n
|
509
|
+
|
510
|
+
# validate: the routine r exists
|
511
|
+
errors << "Routine #{routine_name} not found" unless r
|
512
|
+
|
513
|
+
# raise exception if any error has been found
|
514
|
+
raise "The routine #{routine_name} cannot be run on the node #{node_name}: #{errors.uniq.join(".\n")}" if errors.length > 0
|
515
|
+
|
516
|
+
# connect the node
|
517
|
+
self.logger.logs "Connecting to node #{n.name}... "
|
518
|
+
n.connect
|
519
|
+
self.logger.done
|
520
|
+
|
521
|
+
# run the routine
|
522
|
+
self.logger.logs "Running routine #{r.name}... "
|
523
|
+
r.run(n)
|
524
|
+
self.logger.done
|
525
|
+
|
526
|
+
# disconnect the node
|
527
|
+
self.logger.logs "Disconnecting from node #{n.name}... "
|
528
|
+
n.disconnect
|
529
|
+
self.logger.done
|
530
|
+
|
531
|
+
end # def
|
532
|
+
|
533
|
+
module DB
|
534
|
+
LOCKFILE = './blackstack-deployer.lock'
|
535
|
+
@@checkpoint = nil
|
536
|
+
@@superhuser = nil
|
537
|
+
@@ndb = nil
|
538
|
+
@@folder = nil
|
539
|
+
|
540
|
+
def self.set_checkpoint(s)
|
541
|
+
@@checkpoint = s
|
542
|
+
end
|
543
|
+
|
544
|
+
def self.checkpoint
|
545
|
+
@@checkpoint
|
546
|
+
end
|
547
|
+
|
548
|
+
def self.save_checkpoint
|
549
|
+
File.new('./blackstack-deployer.lock', "w").write(@@checkpoint)
|
550
|
+
end
|
551
|
+
|
552
|
+
def self.load_checkpoint
|
553
|
+
if File.exists?(BlackStack::Deployer::DB::LOCKFILE)
|
554
|
+
@@checkpoint = File.new(BlackStack::Deployer::DB::LOCKFILE, "r").read
|
555
|
+
else
|
556
|
+
@@checkpoint = nil
|
557
|
+
end
|
558
|
+
@@checkpoint
|
559
|
+
end
|
560
|
+
|
561
|
+
def self.connect(s)
|
562
|
+
@@db = Sequel::connect(s)
|
563
|
+
end # def
|
564
|
+
|
565
|
+
def self.set_folder(s)
|
566
|
+
@@folder = s
|
567
|
+
end # def
|
568
|
+
|
569
|
+
|
570
|
+
# Return `true` if the name of the file matches with `/\.transactions\./`, and it doesn't match with `/\.sentences\./`, and the matches with `/\.transactions\./` are no more than one.
|
571
|
+
# Otherwise, return `false`.
|
572
|
+
# This method should not be called directly by user code.
|
573
|
+
def self.is_transactions_file?(filename)
|
574
|
+
filename =~ /\.transactions\./ && filename !~ /\.sentences\./ && filename.scan(/\.transactions\./).size == 1
|
575
|
+
end
|
576
|
+
|
577
|
+
# Return `true` if the name of the file matches with `/\.sentences\./`, and it doesn't match with `/\.transactions\./`, and the matches with `/\.sentences\./` are no more than one.
|
578
|
+
# Otherwise({, return `false`.
|
579
|
+
# This method should not be called directly by user code.
|
580
|
+
def self.is_sentences_file?(filename)
|
581
|
+
filename =~ /\.sentences\./ && filename !~ /\.transactions\./ && filename.scan(/\.sentences\./).size == 1
|
582
|
+
end
|
583
|
+
|
584
|
+
# Method to process an `.sql` file with transactions code, separated by `BEGIN;` and `COMMIT;` statements.
|
585
|
+
# Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
|
586
|
+
# This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.tsql\./`.
|
587
|
+
# This method should not be called directly by user code.
|
588
|
+
def self.execute_transactions(sql)
|
589
|
+
# TODO: Code Me!
|
590
|
+
end # def db_execute_tsql
|
591
|
+
|
592
|
+
# Method to process an `.sql` file with one sql sentence by line.
|
593
|
+
# Reference: https://stackoverflow.com/questions/64066344/import-large-sql-files-with-ruby-sequel-gem
|
594
|
+
# This method is called by `BlackStack::Deployer::db_execute_file` if the filename matches with `/\.sentences\./`.
|
595
|
+
# This method should not be called directly by user code.
|
596
|
+
def self.execute_sentences(sql, chunk_size=200)
|
597
|
+
tlogger = BlackStack::Deployer::logger
|
598
|
+
|
599
|
+
# Fix issue: Ruby `split': invalid byte sequence in UTF-8 (ArgumentError)
|
600
|
+
# Reference: https://stackoverflow.com/questions/11065962/ruby-split-invalid-byte-sequence-in-utf-8-argumenterror
|
601
|
+
#
|
602
|
+
# Fix issue: `PG::SyntaxError: ERROR: at or near "��truncate": syntax error`
|
603
|
+
#
|
604
|
+
sql.encode!('UTF-8', :invalid => :replace, :replace => '')
|
605
|
+
|
606
|
+
# Remove null bytes to avoid error: `String contains null byte`
|
607
|
+
# Reference: https://stackoverflow.com/questions/29320369/coping-with-string-contains-null-byte-sent-from-users
|
608
|
+
sql.gsub!("\u0000", "")
|
609
|
+
|
610
|
+
# Get the array of sentences
|
611
|
+
tlogger.logs "Splitting the sql sentences... "
|
612
|
+
sentences = sql.split(/;/i)
|
613
|
+
tlogger.logf "done (#{sentences.size})"
|
614
|
+
|
615
|
+
# Chunk the array into parts of chunk_size elements
|
616
|
+
# Reference: https://stackoverflow.com/questions/2699584/how-to-split-chunk-a-ruby-array-into-parts-of-x-elements
|
617
|
+
tlogger.logs "Bunlding the array of sentences into chunks of #{chunk_size} each... "
|
618
|
+
chunks = sentences.each_slice(chunk_size).to_a
|
619
|
+
tlogger.logf "done (#{chunks.size})"
|
620
|
+
|
621
|
+
chunk_number = -1
|
622
|
+
chunks.each { |chunk|
|
623
|
+
chunk_number += 1
|
624
|
+
statement = chunk.join(";\n").to_s.strip
|
625
|
+
tlogger.logs "lines #{chunk_size*chunk_number+1} to #{chunk_size*chunk_number+chunk.size} of #{sentences.size}... "
|
626
|
+
begin
|
627
|
+
@@db.execute(statement) #if statement.to_s.strip.size > 0
|
628
|
+
tlogger.done
|
629
|
+
rescue => e
|
630
|
+
tlogger.logf e.to_s
|
631
|
+
raise "Error executing statement: #{statement}\n#{e.message}"
|
632
|
+
end
|
633
|
+
}
|
634
|
+
tlogger.done
|
635
|
+
end # def db_execute_sql_sentences_file
|
636
|
+
|
637
|
+
# Run a series of `.sql` files with updates to the database.
|
638
|
+
#
|
639
|
+
def self.deploy(save_checkpoints=false)
|
640
|
+
tlogger = BlackStack::Deployer::logger
|
641
|
+
# get list of `.sql` files in the directory `sql_path`, with a name higher than `last_filename`, sorted by name.
|
642
|
+
Dir.entries(@@folder).select {
|
643
|
+
|filename| filename =~ /\.sql$/ && filename > @@checkpoint.to_s
|
644
|
+
}.uniq.sort.each { |filename|
|
645
|
+
fullfilename = "#{@@folder}/#{filename}"
|
646
|
+
|
647
|
+
tlogger.logs "#{fullfilename}... "
|
648
|
+
BlackStack::Deployer::DB::execute_sentences( File.open(fullfilename).read )
|
649
|
+
tlogger.done
|
650
|
+
|
651
|
+
tlogger.logs "Updating checkpoint... "
|
652
|
+
BlackStack::Deployer::DB::set_checkpoint filename
|
653
|
+
tlogger.done
|
654
|
+
|
655
|
+
tlogger.logs 'Saving checkpoint... '
|
656
|
+
if save_checkpoints
|
657
|
+
BlackStack::Deployer::DB::save_checkpoint
|
658
|
+
tlogger.done
|
659
|
+
else
|
660
|
+
tlogger.logf 'disabled'
|
661
|
+
end
|
662
|
+
}
|
663
|
+
end # def
|
664
|
+
end # module DB
|
665
|
+
|
666
|
+
# deploying all db-updates and run all routines on all nodes
|
667
|
+
def self.deploy()
|
668
|
+
tlogger = BlackStack::Deployer::logger
|
669
|
+
|
670
|
+
@@nodes.each { |n|
|
671
|
+
tlogger.logs "Node #{n.name}... "
|
672
|
+
n.deploy()
|
673
|
+
tlogger.done
|
674
|
+
}
|
675
|
+
end # def
|
676
|
+
|
677
|
+
end # module Deployer
|
678
|
+
|
670
679
|
end # module BlackStack
|