continuent-tools-core 0.1.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/continuent-tools-core.rb +0 -11
- data/lib/iniparse.rb +66 -0
- data/lib/iniparse/LICENSE +19 -0
- data/lib/iniparse/document.rb +84 -0
- data/lib/iniparse/generator.rb +203 -0
- data/lib/iniparse/line_collection.rb +170 -0
- data/lib/iniparse/lines.rb +327 -0
- data/lib/iniparse/parser.rb +110 -0
- data/lib/tungsten/api.rb +3 -3
- data/lib/tungsten/common.rb +2 -2
- data/lib/tungsten/exec.rb +1 -10
- data/lib/tungsten/iniparse.rb +53 -0
- data/lib/tungsten/install.rb +111 -3
- data/lib/tungsten/properties.rb +11 -19
- data/lib/tungsten/script.rb +124 -2
- data/lib/tungsten/util.rb +94 -1
- metadata +11 -3
data/lib/tungsten/common.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
DEFAULTS = "
|
1
|
+
DEFAULTS = "__defaults__"
|
2
2
|
REPL_RMI_PORT = "repl_rmi_port"
|
3
3
|
MGR_RMI_PORT = "mgr_rmi_port"
|
4
4
|
HOST_ENABLE_REPLICATOR = "host_enable_replicator"
|
@@ -37,7 +37,7 @@ class CommandError < StandardError
|
|
37
37
|
if @errors == ""
|
38
38
|
errors = "No Errors"
|
39
39
|
else
|
40
|
-
errors = "Errors: #{errors}"
|
40
|
+
errors = "Errors: #{@errors}"
|
41
41
|
end
|
42
42
|
|
43
43
|
"Failed: #{command}, RC: #{rc}, Result: #{result}, #{errors}"
|
data/lib/tungsten/exec.rb
CHANGED
@@ -376,7 +376,7 @@ class TungstenUtil
|
|
376
376
|
end
|
377
377
|
|
378
378
|
def get_ssh_command_options
|
379
|
-
opts = ["-A", "-oStrictHostKeyChecking=no"
|
379
|
+
opts = ["-A", "-oStrictHostKeyChecking=no"]
|
380
380
|
@ssh_options.each{
|
381
381
|
|k,v|
|
382
382
|
opts << "-o#{k.to_s()}=#{v}"
|
@@ -415,15 +415,6 @@ class TungstenUtil
|
|
415
415
|
end
|
416
416
|
end
|
417
417
|
|
418
|
-
def get_ssh_port(port = 22)
|
419
|
-
ssh_options = get_ssh_options
|
420
|
-
if ssh_options.has_key?(:port) && ssh_options[:port].to_s != ""
|
421
|
-
ssh_options[:port]
|
422
|
-
else
|
423
|
-
port
|
424
|
-
end
|
425
|
-
end
|
426
|
-
|
427
418
|
def ssh_init_profile_script
|
428
419
|
if @ssh_init_profile_script == nil
|
429
420
|
init_profile_script_parts = []
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'iniparse'
|
2
|
+
|
3
|
+
module IniParse
|
4
|
+
module Lines
|
5
|
+
class BlankOption
|
6
|
+
include Line
|
7
|
+
|
8
|
+
@regex = /^\s*([^=]+) # Option
|
9
|
+
$
|
10
|
+
/x
|
11
|
+
|
12
|
+
attr_accessor :key, :value
|
13
|
+
|
14
|
+
# ==== Parameters
|
15
|
+
# key<String>:: The option key.
|
16
|
+
# value<String>:: The value for this option.
|
17
|
+
# opts<Hash>:: Extra options for the line.
|
18
|
+
#
|
19
|
+
def initialize(key, value = nil, opts = {})
|
20
|
+
super(opts)
|
21
|
+
@key, @value = nil, value
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.parse(line, opts)
|
25
|
+
if m = @regex.match(line)
|
26
|
+
[:option, m[1].strip, nil, opts]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
#######
|
31
|
+
private
|
32
|
+
#######
|
33
|
+
|
34
|
+
def line_contents
|
35
|
+
'%s' % [key]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
module IniParse
|
42
|
+
module Lines
|
43
|
+
class Option
|
44
|
+
def self.typecast(value)
|
45
|
+
value
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
IniParse::Parser.parse_types = [IniParse::Lines::Section,
|
52
|
+
IniParse::Lines::Option, IniParse::Lines::BlankOption,
|
53
|
+
IniParse::Lines::Blank]
|
data/lib/tungsten/install.rb
CHANGED
@@ -75,8 +75,8 @@ class TungstenInstall
|
|
75
75
|
def dataservices
|
76
76
|
ds_list = TU.cmd_result("egrep \"^service.name\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static-* | awk -F \"=\" '{print $2}'").split("\n")
|
77
77
|
|
78
|
-
if use_tpm?()
|
79
|
-
ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices |
|
78
|
+
if use_tpm?() && is_manager?()
|
79
|
+
ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | awk -F \" \" '{print $1}'").split("\n")
|
80
80
|
end
|
81
81
|
|
82
82
|
ds_list.uniq()
|
@@ -93,7 +93,7 @@ class TungstenInstall
|
|
93
93
|
end
|
94
94
|
|
95
95
|
if local_services.size() == 0
|
96
|
-
dataservices()
|
96
|
+
dataservices()[0]
|
97
97
|
else
|
98
98
|
TU.cmd_result("egrep \"^service.name\" #{local_services[0]} | awk -F \"=\" '{print $2}'")
|
99
99
|
end
|
@@ -312,6 +312,114 @@ class TungstenInstall
|
|
312
312
|
end
|
313
313
|
end
|
314
314
|
|
315
|
+
def datasource_type(service, use_direct_extractor = false)
|
316
|
+
if use_direct_extractor == true
|
317
|
+
key = "repl_direct_datasource_type"
|
318
|
+
else
|
319
|
+
key = "repl_datasource_type"
|
320
|
+
end
|
321
|
+
|
322
|
+
setting(setting_key(REPL_SERVICES, service, key))
|
323
|
+
end
|
324
|
+
|
325
|
+
def can_sql?(service, use_direct_extractor = false)
|
326
|
+
case datasource_type(service, use_direct_extractor)
|
327
|
+
when "mysql"
|
328
|
+
return true
|
329
|
+
when "redshift"
|
330
|
+
return true
|
331
|
+
else
|
332
|
+
return false
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
def sql_results(service, sql, use_direct_extractor = false)
|
337
|
+
unless can_sql?(service, use_direct_extractor)
|
338
|
+
raise "Unable to run SQL command for the #{service} dataservice. The datasource type does not support SQL commands."
|
339
|
+
end
|
340
|
+
|
341
|
+
cfg = nil
|
342
|
+
command = nil
|
343
|
+
begin
|
344
|
+
# Disable logging of command results so the password doesn't end up in a log file
|
345
|
+
TU.log_cmd_results?(false)
|
346
|
+
if use_direct_extractor == true
|
347
|
+
url= setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_jdbcqueryurl"))
|
348
|
+
user = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_user"))
|
349
|
+
password = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_password"))
|
350
|
+
else
|
351
|
+
url = setting(setting_key(REPL_SERVICES, service, "repl_datasource_jdbcqueryurl"))
|
352
|
+
user = setting(setting_key(REPL_SERVICES, service, "repl_datasource_user"))
|
353
|
+
password = setting(setting_key(REPL_SERVICES, service, "repl_datasource_password"))
|
354
|
+
end
|
355
|
+
TU.log_cmd_results?(true)
|
356
|
+
|
357
|
+
cfg = Tempfile.new("cnf")
|
358
|
+
cfg.puts("url=#{url}")
|
359
|
+
cfg.puts("user=#{user}")
|
360
|
+
cfg.puts("password=#{password}")
|
361
|
+
cfg.close()
|
362
|
+
|
363
|
+
command = Tempfile.new("query")
|
364
|
+
if sql.is_a?(Array)
|
365
|
+
command.puts(sql.join("\n"))
|
366
|
+
else
|
367
|
+
command.puts(sql)
|
368
|
+
end
|
369
|
+
command.close()
|
370
|
+
|
371
|
+
output = TU.cmd_result("#{self.base_path()}/tungsten-replicator/bin/query -conf #{cfg.path} -file #{command.path}")
|
372
|
+
rescue CommandError => ce
|
373
|
+
TU.debug(ce)
|
374
|
+
raise "There was an error processing the query: #{ce.result}"
|
375
|
+
ensure
|
376
|
+
if cfg != nil
|
377
|
+
cfg.close()
|
378
|
+
cfg.unlink()
|
379
|
+
end
|
380
|
+
|
381
|
+
if command != nil
|
382
|
+
command.close()
|
383
|
+
command.unlink()
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
results = JSON.parse(output)
|
388
|
+
results.each {
|
389
|
+
|result|
|
390
|
+
if result["rc"] == 0
|
391
|
+
result["error"] = nil
|
392
|
+
else
|
393
|
+
obj = CommandError.new(result["statement"], result["rc"], result["results"], result["error"])
|
394
|
+
result["error"] = obj
|
395
|
+
end
|
396
|
+
}
|
397
|
+
results
|
398
|
+
end
|
399
|
+
|
400
|
+
def sql_result(service, sql, use_direct_extractor = false)
|
401
|
+
results = sql_results(service, sql, use_direct_extractor)
|
402
|
+
|
403
|
+
if results.size() == 0
|
404
|
+
raise "No results were returned by #{sql}"
|
405
|
+
end
|
406
|
+
|
407
|
+
if results[0]["error"] == nil
|
408
|
+
return results[0]["results"][0]
|
409
|
+
else
|
410
|
+
raise results[0]["error"]
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
def check_sql_results(results)
|
415
|
+
results.each{
|
416
|
+
|r|
|
417
|
+
if r["error"] != nil
|
418
|
+
raise r["error"]
|
419
|
+
end
|
420
|
+
}
|
421
|
+
end
|
422
|
+
|
315
423
|
def self.get(path)
|
316
424
|
@@instances ||= {}
|
317
425
|
unless @@instances.has_key?(path)
|
data/lib/tungsten/properties.rb
CHANGED
@@ -13,12 +13,20 @@ class Properties
|
|
13
13
|
# Initialize with some base values.
|
14
14
|
def initialize
|
15
15
|
@props = {}
|
16
|
+
|
17
|
+
# These hashes are used to prevent loops where the prompt handler needs
|
18
|
+
# to get additional information from the configuration
|
16
19
|
@in_prompt_handler = {}
|
17
20
|
@in_template_value_prompt_handler = {}
|
21
|
+
|
22
|
+
# The prompt handler provides a structuring for finding values
|
23
|
+
# that do no exist in the props Hash. The prompt handler will calculate
|
24
|
+
# default values or values based on parent defaults.
|
18
25
|
@use_prompt_handler = false
|
19
26
|
@prompt_handler = nil
|
27
|
+
|
28
|
+
# This is used for translating the configuration from 1.3 deployments
|
20
29
|
@force_json = true
|
21
|
-
@debug = false
|
22
30
|
end
|
23
31
|
|
24
32
|
def initialize_copy(source)
|
@@ -102,14 +110,6 @@ class Properties
|
|
102
110
|
self.props = {}
|
103
111
|
end
|
104
112
|
|
105
|
-
def debug(v = nil)
|
106
|
-
if v != nil
|
107
|
-
@debug = v
|
108
|
-
end
|
109
|
-
|
110
|
-
v
|
111
|
-
end
|
112
|
-
|
113
113
|
def import(properties_obj = {})
|
114
114
|
if properties_obj.instance_of?(Hash)
|
115
115
|
self.props = properties_obj
|
@@ -292,11 +292,7 @@ class Properties
|
|
292
292
|
end
|
293
293
|
|
294
294
|
# Get the config file value for a property.
|
295
|
-
def getTemplateValue(key
|
296
|
-
if transform_values_method == nil
|
297
|
-
transform_values_method = method(:blank_transform_values_method)
|
298
|
-
end
|
299
|
-
|
295
|
+
def getTemplateValue(key)
|
300
296
|
if key.is_a?(String)
|
301
297
|
key_string = key
|
302
298
|
key = key.split('.')
|
@@ -313,7 +309,7 @@ class Properties
|
|
313
309
|
begin
|
314
310
|
@in_template_value_prompt_handler[key_string] = true
|
315
311
|
|
316
|
-
value = getPromptHandler().find_template_value(keys
|
312
|
+
value = getPromptHandler().find_template_value(keys)
|
317
313
|
|
318
314
|
@in_template_value_prompt_handler[key_string] = false
|
319
315
|
rescue IgnoreError
|
@@ -363,10 +359,6 @@ class Properties
|
|
363
359
|
value
|
364
360
|
end
|
365
361
|
|
366
|
-
def blank_transform_values_method(matches)
|
367
|
-
""
|
368
|
-
end
|
369
|
-
|
370
362
|
# Get the property value or return the default if nil
|
371
363
|
def getPropertyOr(key, default = "")
|
372
364
|
value = getProperty(key)
|
data/lib/tungsten/script.rb
CHANGED
@@ -28,6 +28,9 @@ module TungstenScript
|
|
28
28
|
# Does this script required to run against an installed Tungsten directory
|
29
29
|
@require_installed_directory = true
|
30
30
|
|
31
|
+
# Should unparsed arguments cause an error
|
32
|
+
@allow_unparsed_arguments = false
|
33
|
+
|
31
34
|
# Definition of each command that this script will support
|
32
35
|
@command_definitions = {}
|
33
36
|
|
@@ -44,6 +47,9 @@ module TungstenScript
|
|
44
47
|
# The collected option values from the script input
|
45
48
|
@options = {}
|
46
49
|
|
50
|
+
# Parameters loaded from INI files to be parsed
|
51
|
+
@ini_parameters = []
|
52
|
+
|
47
53
|
TU.debug("Begin #{$0} #{ARGV.join(' ')}")
|
48
54
|
|
49
55
|
begin
|
@@ -60,8 +66,11 @@ module TungstenScript
|
|
60
66
|
cleanup(0)
|
61
67
|
end
|
62
68
|
|
69
|
+
# Load parameters from the available INI files
|
70
|
+
load_ini_files()
|
71
|
+
# Parse parameters loaded from the INI files and on command line
|
63
72
|
parse_options()
|
64
|
-
|
73
|
+
|
65
74
|
if @options[:autocomplete] == true
|
66
75
|
display_autocomplete()
|
67
76
|
cleanup(0)
|
@@ -133,6 +142,15 @@ module TungstenScript
|
|
133
142
|
return @options[option_key]
|
134
143
|
end
|
135
144
|
|
145
|
+
# Set the value for option_key if it has not been set
|
146
|
+
def opt_default(option_key, default_value)
|
147
|
+
if opt(option_key) == nil
|
148
|
+
opt(option_key, default_value)
|
149
|
+
end
|
150
|
+
|
151
|
+
opt(option_key)
|
152
|
+
end
|
153
|
+
|
136
154
|
def add_command(command_key, definition)
|
137
155
|
begin
|
138
156
|
command_key = command_key.to_sym()
|
@@ -193,6 +211,69 @@ module TungstenScript
|
|
193
211
|
@option_definitions[option_key][:default] = default
|
194
212
|
end
|
195
213
|
|
214
|
+
def load_ini_files
|
215
|
+
# Calculate the INI section name to use
|
216
|
+
section_names = [script_name()]
|
217
|
+
matches = script_name().to_s().match("tungsten_(.*)")
|
218
|
+
if matches && matches.size() > 0
|
219
|
+
script_ini_file = "#{matches[1]}.ini"
|
220
|
+
section_names << matches[1]
|
221
|
+
else
|
222
|
+
script_ini_file = nil
|
223
|
+
end
|
224
|
+
|
225
|
+
load_ini_parameters("/etc/tungsten/scripts.ini",
|
226
|
+
section_names)
|
227
|
+
|
228
|
+
if script_ini_file != nil
|
229
|
+
load_ini_parameters("/etc/tungsten/#{script_ini_file}",
|
230
|
+
["__anonymous__"] + section_names)
|
231
|
+
end
|
232
|
+
|
233
|
+
# Add these arguments to the beginging of the TungstenUtil stack
|
234
|
+
# When the script processes command line options it will read these
|
235
|
+
# and then be overwritten by and command line options.
|
236
|
+
TU.remaining_arguments = @ini_parameters + TU.remaining_arguments
|
237
|
+
end
|
238
|
+
|
239
|
+
# Convert the parsed INI contents into the command line argument style
|
240
|
+
def load_ini_parameters(file, section_name)
|
241
|
+
unless File.exists?(file)
|
242
|
+
return
|
243
|
+
end
|
244
|
+
|
245
|
+
unless section_name.is_a?(Array)
|
246
|
+
section_name = [section_name]
|
247
|
+
end
|
248
|
+
section_name.delete_if{|n| n.to_s() == ""}
|
249
|
+
|
250
|
+
if section_name.size() == 0
|
251
|
+
return
|
252
|
+
end
|
253
|
+
|
254
|
+
parameters = TU.parse_ini_file(file, false)
|
255
|
+
section_name.each{
|
256
|
+
|section|
|
257
|
+
if section.to_s() == ""
|
258
|
+
next
|
259
|
+
end
|
260
|
+
|
261
|
+
unless parameters.has_key?(section)
|
262
|
+
next
|
263
|
+
end
|
264
|
+
|
265
|
+
parameters[section].each{
|
266
|
+
|line|
|
267
|
+
# Single character parameters get a single dash
|
268
|
+
if line.length() == 1
|
269
|
+
@ini_parameters << "-#{line}"
|
270
|
+
else
|
271
|
+
@ini_parameters << "--#{line}"
|
272
|
+
end
|
273
|
+
}
|
274
|
+
}
|
275
|
+
end
|
276
|
+
|
196
277
|
def parse_options
|
197
278
|
opts = OptionParser.new()
|
198
279
|
|
@@ -269,9 +350,26 @@ module TungstenScript
|
|
269
350
|
end
|
270
351
|
end
|
271
352
|
|
353
|
+
unless allow_unparsed_arguments?()
|
354
|
+
unless TU.remaining_arguments.size == 0
|
355
|
+
TU.error("Unable to parse the following arguments: #{TU.remaining_arguments.join(' ')}")
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
272
359
|
if require_command?() && @command_definitions.size() > 0 && @command == nil
|
273
360
|
TU.error("A command was not given for this script. Valid commands are #{@command_definitions.keys().join(', ')} and must be the first argument.")
|
274
361
|
end
|
362
|
+
|
363
|
+
@option_definitions.each{
|
364
|
+
|option_key,definition|
|
365
|
+
|
366
|
+
if definition[:required] == true
|
367
|
+
if opt(option_key).to_s() == ""
|
368
|
+
arg = definition[:on][0].split(" ")[0]
|
369
|
+
TU.error("Missing value for the #{arg} option")
|
370
|
+
end
|
371
|
+
end
|
372
|
+
}
|
275
373
|
end
|
276
374
|
|
277
375
|
def script_name
|
@@ -373,6 +471,14 @@ module TungstenScript
|
|
373
471
|
@require_installed_directory
|
374
472
|
end
|
375
473
|
|
474
|
+
def allow_unparsed_arguments?(v = nil)
|
475
|
+
if (v != nil)
|
476
|
+
@allow_unparsed_arguments = v
|
477
|
+
end
|
478
|
+
|
479
|
+
@allow_unparsed_arguments
|
480
|
+
end
|
481
|
+
|
376
482
|
def require_command?
|
377
483
|
true
|
378
484
|
end
|
@@ -394,8 +500,16 @@ module TungstenScript
|
|
394
500
|
end
|
395
501
|
|
396
502
|
def cleanup(code = 0)
|
503
|
+
if code != 0
|
504
|
+
log_path = TU.log().path()
|
505
|
+
if log_path.to_s() != "" && File.exist?(log_path)
|
506
|
+
TU.notice("See #{script_log_path()} for more information")
|
507
|
+
end
|
508
|
+
end
|
509
|
+
|
397
510
|
TU.debug("Finish #{$0} #{ARGV.join(' ')}")
|
398
511
|
TU.debug("RC: #{code}")
|
512
|
+
|
399
513
|
TU.exit(code)
|
400
514
|
end
|
401
515
|
|
@@ -770,6 +884,14 @@ module MySQLServiceScript
|
|
770
884
|
"mysqldump --defaults-file=#{@options[:my_cnf]} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]} --opt --single-transaction --all-databases --add-drop-database --master-data=2"
|
771
885
|
end
|
772
886
|
|
887
|
+
def get_innobackupex_path()
|
888
|
+
path = TU.which("innobackupex-1.5.1")
|
889
|
+
if path.nil?
|
890
|
+
path = TU.which("innobackupex")
|
891
|
+
end
|
892
|
+
return path
|
893
|
+
end
|
894
|
+
|
773
895
|
def get_xtrabackup_command
|
774
896
|
# Use the configured my.cnf file, or the additional config file
|
775
897
|
# if we created one
|
@@ -779,7 +901,7 @@ module MySQLServiceScript
|
|
779
901
|
defaults_file = @options[:extra_mysql_defaults_file].path()
|
780
902
|
end
|
781
903
|
|
782
|
-
"
|
904
|
+
"#{get_innobackupex_path()} --defaults-file=#{defaults_file} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]}"
|
783
905
|
end
|
784
906
|
|
785
907
|
def xtrabackup_supports_argument(arg)
|