continuent-tools-core 0.1.6 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- DEFAULTS = "defaults"
1
+ DEFAULTS = "__defaults__"
2
2
  REPL_RMI_PORT = "repl_rmi_port"
3
3
  MGR_RMI_PORT = "mgr_rmi_port"
4
4
  HOST_ENABLE_REPLICATOR = "host_enable_replicator"
@@ -37,7 +37,7 @@ class CommandError < StandardError
37
37
  if @errors == ""
38
38
  errors = "No Errors"
39
39
  else
40
- errors = "Errors: #{errors}"
40
+ errors = "Errors: #{@errors}"
41
41
  end
42
42
 
43
43
  "Failed: #{command}, RC: #{rc}, Result: #{result}, #{errors}"
data/lib/tungsten/exec.rb CHANGED
@@ -376,7 +376,7 @@ class TungstenUtil
376
376
  end
377
377
 
378
378
  def get_ssh_command_options
379
- opts = ["-A", "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null"]
379
+ opts = ["-A", "-oStrictHostKeyChecking=no"]
380
380
  @ssh_options.each{
381
381
  |k,v|
382
382
  opts << "-o#{k.to_s()}=#{v}"
@@ -415,15 +415,6 @@ class TungstenUtil
415
415
  end
416
416
  end
417
417
 
418
- def get_ssh_port(port = 22)
419
- ssh_options = get_ssh_options
420
- if ssh_options.has_key?(:port) && ssh_options[:port].to_s != ""
421
- ssh_options[:port]
422
- else
423
- port
424
- end
425
- end
426
-
427
418
  def ssh_init_profile_script
428
419
  if @ssh_init_profile_script == nil
429
420
  init_profile_script_parts = []
@@ -0,0 +1,53 @@
1
+ require 'iniparse'
2
+
3
+ module IniParse
4
+ module Lines
5
+ class BlankOption
6
+ include Line
7
+
8
+ @regex = /^\s*([^=]+) # Option
9
+ $
10
+ /x
11
+
12
+ attr_accessor :key, :value
13
+
14
+ # ==== Parameters
15
+ # key<String>:: The option key.
16
+ # value<String>:: The value for this option.
17
+ # opts<Hash>:: Extra options for the line.
18
+ #
19
+ def initialize(key, value = nil, opts = {})
20
+ super(opts)
21
+ @key, @value = nil, value
22
+ end
23
+
24
+ def self.parse(line, opts)
25
+ if m = @regex.match(line)
26
+ [:option, m[1].strip, nil, opts]
27
+ end
28
+ end
29
+
30
+ #######
31
+ private
32
+ #######
33
+
34
+ def line_contents
35
+ '%s' % [key]
36
+ end
37
+ end
38
+ end
39
+ end
40
+
41
+ module IniParse
42
+ module Lines
43
+ class Option
44
+ def self.typecast(value)
45
+ value
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ IniParse::Parser.parse_types = [IniParse::Lines::Section,
52
+ IniParse::Lines::Option, IniParse::Lines::BlankOption,
53
+ IniParse::Lines::Blank]
@@ -75,8 +75,8 @@ class TungstenInstall
75
75
  def dataservices
76
76
  ds_list = TU.cmd_result("egrep \"^service.name\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static-* | awk -F \"=\" '{print $2}'").split("\n")
77
77
 
78
- if use_tpm?()
79
- ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | grep COMPOSITE | awk -F \" \" '{print $1}'").split("\n")
78
+ if use_tpm?() && is_manager?()
79
+ ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | awk -F \" \" '{print $1}'").split("\n")
80
80
  end
81
81
 
82
82
  ds_list.uniq()
@@ -93,7 +93,7 @@ class TungstenInstall
93
93
  end
94
94
 
95
95
  if local_services.size() == 0
96
- dataservices().get(0)
96
+ dataservices()[0]
97
97
  else
98
98
  TU.cmd_result("egrep \"^service.name\" #{local_services[0]} | awk -F \"=\" '{print $2}'")
99
99
  end
@@ -312,6 +312,114 @@ class TungstenInstall
312
312
  end
313
313
  end
314
314
 
315
+ def datasource_type(service, use_direct_extractor = false)
316
+ if use_direct_extractor == true
317
+ key = "repl_direct_datasource_type"
318
+ else
319
+ key = "repl_datasource_type"
320
+ end
321
+
322
+ setting(setting_key(REPL_SERVICES, service, key))
323
+ end
324
+
325
+ def can_sql?(service, use_direct_extractor = false)
326
+ case datasource_type(service, use_direct_extractor)
327
+ when "mysql"
328
+ return true
329
+ when "redshift"
330
+ return true
331
+ else
332
+ return false
333
+ end
334
+ end
335
+
336
+ def sql_results(service, sql, use_direct_extractor = false)
337
+ unless can_sql?(service, use_direct_extractor)
338
+ raise "Unable to run SQL command for the #{service} dataservice. The datasource type does not support SQL commands."
339
+ end
340
+
341
+ cfg = nil
342
+ command = nil
343
+ begin
344
+ # Disable logging of command results so the password doesn't end up in a log file
345
+ TU.log_cmd_results?(false)
346
+ if use_direct_extractor == true
347
+ url= setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_jdbcqueryurl"))
348
+ user = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_user"))
349
+ password = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_password"))
350
+ else
351
+ url = setting(setting_key(REPL_SERVICES, service, "repl_datasource_jdbcqueryurl"))
352
+ user = setting(setting_key(REPL_SERVICES, service, "repl_datasource_user"))
353
+ password = setting(setting_key(REPL_SERVICES, service, "repl_datasource_password"))
354
+ end
355
+ TU.log_cmd_results?(true)
356
+
357
+ cfg = Tempfile.new("cnf")
358
+ cfg.puts("url=#{url}")
359
+ cfg.puts("user=#{user}")
360
+ cfg.puts("password=#{password}")
361
+ cfg.close()
362
+
363
+ command = Tempfile.new("query")
364
+ if sql.is_a?(Array)
365
+ command.puts(sql.join("\n"))
366
+ else
367
+ command.puts(sql)
368
+ end
369
+ command.close()
370
+
371
+ output = TU.cmd_result("#{self.base_path()}/tungsten-replicator/bin/query -conf #{cfg.path} -file #{command.path}")
372
+ rescue CommandError => ce
373
+ TU.debug(ce)
374
+ raise "There was an error processing the query: #{ce.result}"
375
+ ensure
376
+ if cfg != nil
377
+ cfg.close()
378
+ cfg.unlink()
379
+ end
380
+
381
+ if command != nil
382
+ command.close()
383
+ command.unlink()
384
+ end
385
+ end
386
+
387
+ results = JSON.parse(output)
388
+ results.each {
389
+ |result|
390
+ if result["rc"] == 0
391
+ result["error"] = nil
392
+ else
393
+ obj = CommandError.new(result["statement"], result["rc"], result["results"], result["error"])
394
+ result["error"] = obj
395
+ end
396
+ }
397
+ results
398
+ end
399
+
400
+ def sql_result(service, sql, use_direct_extractor = false)
401
+ results = sql_results(service, sql, use_direct_extractor)
402
+
403
+ if results.size() == 0
404
+ raise "No results were returned by #{sql}"
405
+ end
406
+
407
+ if results[0]["error"] == nil
408
+ return results[0]["results"][0]
409
+ else
410
+ raise results[0]["error"]
411
+ end
412
+ end
413
+
414
+ def check_sql_results(results)
415
+ results.each{
416
+ |r|
417
+ if r["error"] != nil
418
+ raise r["error"]
419
+ end
420
+ }
421
+ end
422
+
315
423
  def self.get(path)
316
424
  @@instances ||= {}
317
425
  unless @@instances.has_key?(path)
@@ -13,12 +13,20 @@ class Properties
13
13
  # Initialize with some base values.
14
14
  def initialize
15
15
  @props = {}
16
+
17
+ # These hashes are used to prevent loops where the prompt handler needs
18
+ # to get additional information from the configuration
16
19
  @in_prompt_handler = {}
17
20
  @in_template_value_prompt_handler = {}
21
+
22
+ # The prompt handler provides a structuring for finding values
23
+ # that do no exist in the props Hash. The prompt handler will calculate
24
+ # default values or values based on parent defaults.
18
25
  @use_prompt_handler = false
19
26
  @prompt_handler = nil
27
+
28
+ # This is used for translating the configuration from 1.3 deployments
20
29
  @force_json = true
21
- @debug = false
22
30
  end
23
31
 
24
32
  def initialize_copy(source)
@@ -102,14 +110,6 @@ class Properties
102
110
  self.props = {}
103
111
  end
104
112
 
105
- def debug(v = nil)
106
- if v != nil
107
- @debug = v
108
- end
109
-
110
- v
111
- end
112
-
113
113
  def import(properties_obj = {})
114
114
  if properties_obj.instance_of?(Hash)
115
115
  self.props = properties_obj
@@ -292,11 +292,7 @@ class Properties
292
292
  end
293
293
 
294
294
  # Get the config file value for a property.
295
- def getTemplateValue(key, transform_values_method = nil)
296
- if transform_values_method == nil
297
- transform_values_method = method(:blank_transform_values_method)
298
- end
299
-
295
+ def getTemplateValue(key)
300
296
  if key.is_a?(String)
301
297
  key_string = key
302
298
  key = key.split('.')
@@ -313,7 +309,7 @@ class Properties
313
309
  begin
314
310
  @in_template_value_prompt_handler[key_string] = true
315
311
 
316
- value = getPromptHandler().find_template_value(keys, transform_values_method)
312
+ value = getPromptHandler().find_template_value(keys)
317
313
 
318
314
  @in_template_value_prompt_handler[key_string] = false
319
315
  rescue IgnoreError
@@ -363,10 +359,6 @@ class Properties
363
359
  value
364
360
  end
365
361
 
366
- def blank_transform_values_method(matches)
367
- ""
368
- end
369
-
370
362
  # Get the property value or return the default if nil
371
363
  def getPropertyOr(key, default = "")
372
364
  value = getProperty(key)
@@ -28,6 +28,9 @@ module TungstenScript
28
28
  # Does this script required to run against an installed Tungsten directory
29
29
  @require_installed_directory = true
30
30
 
31
+ # Should unparsed arguments cause an error
32
+ @allow_unparsed_arguments = false
33
+
31
34
  # Definition of each command that this script will support
32
35
  @command_definitions = {}
33
36
 
@@ -44,6 +47,9 @@ module TungstenScript
44
47
  # The collected option values from the script input
45
48
  @options = {}
46
49
 
50
+ # Parameters loaded from INI files to be parsed
51
+ @ini_parameters = []
52
+
47
53
  TU.debug("Begin #{$0} #{ARGV.join(' ')}")
48
54
 
49
55
  begin
@@ -60,8 +66,11 @@ module TungstenScript
60
66
  cleanup(0)
61
67
  end
62
68
 
69
+ # Load parameters from the available INI files
70
+ load_ini_files()
71
+ # Parse parameters loaded from the INI files and on command line
63
72
  parse_options()
64
-
73
+
65
74
  if @options[:autocomplete] == true
66
75
  display_autocomplete()
67
76
  cleanup(0)
@@ -133,6 +142,15 @@ module TungstenScript
133
142
  return @options[option_key]
134
143
  end
135
144
 
145
+ # Set the value for option_key if it has not been set
146
+ def opt_default(option_key, default_value)
147
+ if opt(option_key) == nil
148
+ opt(option_key, default_value)
149
+ end
150
+
151
+ opt(option_key)
152
+ end
153
+
136
154
  def add_command(command_key, definition)
137
155
  begin
138
156
  command_key = command_key.to_sym()
@@ -193,6 +211,69 @@ module TungstenScript
193
211
  @option_definitions[option_key][:default] = default
194
212
  end
195
213
 
214
+ def load_ini_files
215
+ # Calculate the INI section name to use
216
+ section_names = [script_name()]
217
+ matches = script_name().to_s().match("tungsten_(.*)")
218
+ if matches && matches.size() > 0
219
+ script_ini_file = "#{matches[1]}.ini"
220
+ section_names << matches[1]
221
+ else
222
+ script_ini_file = nil
223
+ end
224
+
225
+ load_ini_parameters("/etc/tungsten/scripts.ini",
226
+ section_names)
227
+
228
+ if script_ini_file != nil
229
+ load_ini_parameters("/etc/tungsten/#{script_ini_file}",
230
+ ["__anonymous__"] + section_names)
231
+ end
232
+
233
+ # Add these arguments to the beginging of the TungstenUtil stack
234
+ # When the script processes command line options it will read these
235
+ # and then be overwritten by and command line options.
236
+ TU.remaining_arguments = @ini_parameters + TU.remaining_arguments
237
+ end
238
+
239
+ # Convert the parsed INI contents into the command line argument style
240
+ def load_ini_parameters(file, section_name)
241
+ unless File.exists?(file)
242
+ return
243
+ end
244
+
245
+ unless section_name.is_a?(Array)
246
+ section_name = [section_name]
247
+ end
248
+ section_name.delete_if{|n| n.to_s() == ""}
249
+
250
+ if section_name.size() == 0
251
+ return
252
+ end
253
+
254
+ parameters = TU.parse_ini_file(file, false)
255
+ section_name.each{
256
+ |section|
257
+ if section.to_s() == ""
258
+ next
259
+ end
260
+
261
+ unless parameters.has_key?(section)
262
+ next
263
+ end
264
+
265
+ parameters[section].each{
266
+ |line|
267
+ # Single character parameters get a single dash
268
+ if line.length() == 1
269
+ @ini_parameters << "-#{line}"
270
+ else
271
+ @ini_parameters << "--#{line}"
272
+ end
273
+ }
274
+ }
275
+ end
276
+
196
277
  def parse_options
197
278
  opts = OptionParser.new()
198
279
 
@@ -269,9 +350,26 @@ module TungstenScript
269
350
  end
270
351
  end
271
352
 
353
+ unless allow_unparsed_arguments?()
354
+ unless TU.remaining_arguments.size == 0
355
+ TU.error("Unable to parse the following arguments: #{TU.remaining_arguments.join(' ')}")
356
+ end
357
+ end
358
+
272
359
  if require_command?() && @command_definitions.size() > 0 && @command == nil
273
360
  TU.error("A command was not given for this script. Valid commands are #{@command_definitions.keys().join(', ')} and must be the first argument.")
274
361
  end
362
+
363
+ @option_definitions.each{
364
+ |option_key,definition|
365
+
366
+ if definition[:required] == true
367
+ if opt(option_key).to_s() == ""
368
+ arg = definition[:on][0].split(" ")[0]
369
+ TU.error("Missing value for the #{arg} option")
370
+ end
371
+ end
372
+ }
275
373
  end
276
374
 
277
375
  def script_name
@@ -373,6 +471,14 @@ module TungstenScript
373
471
  @require_installed_directory
374
472
  end
375
473
 
474
+ def allow_unparsed_arguments?(v = nil)
475
+ if (v != nil)
476
+ @allow_unparsed_arguments = v
477
+ end
478
+
479
+ @allow_unparsed_arguments
480
+ end
481
+
376
482
  def require_command?
377
483
  true
378
484
  end
@@ -394,8 +500,16 @@ module TungstenScript
394
500
  end
395
501
 
396
502
  def cleanup(code = 0)
503
+ if code != 0
504
+ log_path = TU.log().path()
505
+ if log_path.to_s() != "" && File.exist?(log_path)
506
+ TU.notice("See #{script_log_path()} for more information")
507
+ end
508
+ end
509
+
397
510
  TU.debug("Finish #{$0} #{ARGV.join(' ')}")
398
511
  TU.debug("RC: #{code}")
512
+
399
513
  TU.exit(code)
400
514
  end
401
515
 
@@ -770,6 +884,14 @@ module MySQLServiceScript
770
884
  "mysqldump --defaults-file=#{@options[:my_cnf]} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]} --opt --single-transaction --all-databases --add-drop-database --master-data=2"
771
885
  end
772
886
 
887
+ def get_innobackupex_path()
888
+ path = TU.which("innobackupex-1.5.1")
889
+ if path.nil?
890
+ path = TU.which("innobackupex")
891
+ end
892
+ return path
893
+ end
894
+
773
895
  def get_xtrabackup_command
774
896
  # Use the configured my.cnf file, or the additional config file
775
897
  # if we created one
@@ -779,7 +901,7 @@ module MySQLServiceScript
779
901
  defaults_file = @options[:extra_mysql_defaults_file].path()
780
902
  end
781
903
 
782
- "innobackupex-1.5.1 --defaults-file=#{defaults_file} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]}"
904
+ "#{get_innobackupex_path()} --defaults-file=#{defaults_file} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]}"
783
905
  end
784
906
 
785
907
  def xtrabackup_supports_argument(arg)