continuent-tools-core 0.1.6 → 0.5.0

data/lib/tungsten/common.rb

@@ -1,4 +1,4 @@
-DEFAULTS = "defaults"
+DEFAULTS = "__defaults__"
 REPL_RMI_PORT = "repl_rmi_port"
 MGR_RMI_PORT = "mgr_rmi_port"
 HOST_ENABLE_REPLICATOR = "host_enable_replicator"
@@ -37,7 +37,7 @@ class CommandError < StandardError
     if @errors == ""
       errors = "No Errors"
     else
-      errors = "Errors: #{errors}"
+      errors = "Errors: #{@errors}"
     end
     
     "Failed: #{command}, RC: #{rc}, Result: #{result}, #{errors}"

data/lib/tungsten/exec.rb

@@ -376,7 +376,7 @@ class TungstenUtil
   end
   
   def get_ssh_command_options
-    opts = ["-A", "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null"]
+    opts = ["-A", "-oStrictHostKeyChecking=no"]
     @ssh_options.each{
       |k,v|
       opts << "-o#{k.to_s()}=#{v}"
@@ -415,15 +415,6 @@ class TungstenUtil
     end
   end
   
-  def get_ssh_port(port = 22)
-    ssh_options = get_ssh_options
-    if ssh_options.has_key?(:port) && ssh_options[:port].to_s != ""
-      ssh_options[:port]
-    else
-      port
-    end
-  end
-  
   def ssh_init_profile_script
     if @ssh_init_profile_script == nil
       init_profile_script_parts = []

data/lib/tungsten/iniparse.rb

@@ -0,0 +1,53 @@
+require 'iniparse'
+
+module IniParse
+  module Lines
+    class BlankOption
+      include Line
+
+      @regex = /^\s*([^=]+)  # Option
+                 $
+               /x
+
+      attr_accessor :key, :value
+
+      # ==== Parameters
+      # key<String>::   The option key.
+      # value<String>:: The value for this option.
+      # opts<Hash>::    Extra options for the line.
+      #
+      def initialize(key, value = nil, opts = {})
+        super(opts)
+        @key, @value = nil, value
+      end
+
+      def self.parse(line, opts)
+        if m = @regex.match(line)
+          [:option, m[1].strip, nil, opts]
+        end
+      end
+
+      #######
+      private
+      #######
+
+      def line_contents
+        '%s' % [key]
+      end
+    end
+  end
+end
+
+module IniParse
+  module Lines
+    class Option
+      def self.typecast(value)
+        value
+      end
+    end
+  end
+end
+
+IniParse::Parser.parse_types = [IniParse::Lines::Section,
+  IniParse::Lines::Option, IniParse::Lines::BlankOption, 
+  IniParse::Lines::Blank]

data/lib/tungsten/install.rb

@@ -75,8 +75,8 @@ class TungstenInstall
   def dataservices
     ds_list = TU.cmd_result("egrep \"^service.name\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static-* | awk -F \"=\" '{print $2}'").split("\n")
     
-    if use_tpm?()
-      ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | grep COMPOSITE | awk -F \" \" '{print $1}'").split("\n")
+    if use_tpm?() && is_manager?()
+      ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | awk -F \" \" '{print $1}'").split("\n")
     end
     
     ds_list.uniq()
@@ -93,7 +93,7 @@ class TungstenInstall
       end
       
       if local_services.size() == 0
-        dataservices().get(0)
+        dataservices()[0]
       else
         TU.cmd_result("egrep \"^service.name\" #{local_services[0]} | awk -F \"=\" '{print $2}'")
       end
@@ -312,6 +312,114 @@ class TungstenInstall
     end
   end
   
+  def datasource_type(service, use_direct_extractor = false)
+    if use_direct_extractor == true
+      key = "repl_direct_datasource_type"
+    else
+      key = "repl_datasource_type"
+    end
+    
+    setting(setting_key(REPL_SERVICES, service, key))
+  end
+  
+  def can_sql?(service, use_direct_extractor = false)
+    case datasource_type(service, use_direct_extractor)
+    when "mysql"
+      return true
+    when "redshift"
+      return true
+    else
+      return false
+    end
+  end
+  
+  def sql_results(service, sql, use_direct_extractor = false)
+    unless can_sql?(service, use_direct_extractor)
+      raise "Unable to run SQL command for the #{service} dataservice. The datasource type does not support SQL commands."
+    end
+    
+    cfg = nil
+    command = nil
+    begin
+      # Disable logging of command results so the password doesn't end up in a log file
+      TU.log_cmd_results?(false)
+      if use_direct_extractor == true
+        url= setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_jdbcqueryurl"))
+        user = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_user"))
+        password = setting(setting_key(REPL_SERVICES, service, "repl_direct_datasource_password"))
+      else
+        url = setting(setting_key(REPL_SERVICES, service, "repl_datasource_jdbcqueryurl"))
+        user = setting(setting_key(REPL_SERVICES, service, "repl_datasource_user"))
+        password = setting(setting_key(REPL_SERVICES, service, "repl_datasource_password"))
+      end
+      TU.log_cmd_results?(true)
+      
+      cfg = Tempfile.new("cnf")
+      cfg.puts("url=#{url}")
+      cfg.puts("user=#{user}")
+      cfg.puts("password=#{password}")
+      cfg.close()
+      
+      command = Tempfile.new("query")
+      if sql.is_a?(Array)
+        command.puts(sql.join("\n"))
+      else
+        command.puts(sql)
+      end
+      command.close()
+      
+      output = TU.cmd_result("#{self.base_path()}/tungsten-replicator/bin/query -conf #{cfg.path} -file #{command.path}")
+    rescue CommandError => ce
+      TU.debug(ce)
+      raise "There was an error processing the query: #{ce.result}"
+    ensure
+      if cfg != nil
+        cfg.close()
+        cfg.unlink()
+      end
+      
+      if command != nil
+        command.close()
+        command.unlink()
+      end
+    end
+    
+    results = JSON.parse(output)
+    results.each {
+      |result|
+      if result["rc"] == 0
+        result["error"] = nil
+      else
+        obj = CommandError.new(result["statement"], result["rc"], result["results"], result["error"])
+        result["error"] = obj
+      end
+    }
+    results
+  end
+  
+  def sql_result(service, sql, use_direct_extractor = false)
+    results = sql_results(service, sql, use_direct_extractor)
+    
+    if results.size() == 0
+      raise "No results were returned by #{sql}"
+    end
+    
+    if results[0]["error"] == nil
+      return results[0]["results"][0]
+    else
+      raise results[0]["error"]
+    end
+  end
+  
+  def check_sql_results(results)
+    results.each{
+      |r|
+      if r["error"] != nil
+        raise r["error"]
+      end
+    }
+  end
+  
   def self.get(path)
     @@instances ||= {}
     unless @@instances.has_key?(path)

data/lib/tungsten/properties.rb

@@ -13,12 +13,20 @@ class Properties
   # Initialize with some base values. 
   def initialize
     @props = {}
+    
+    # These hashes are used to prevent loops where the prompt handler needs
+    # to get additional information from the configuration
     @in_prompt_handler = {}
     @in_template_value_prompt_handler = {}
+    
+    # The prompt handler provides a structuring for finding values
+    # that do no exist in the props Hash. The prompt handler will calculate
+    # default values or values based on parent defaults.
     @use_prompt_handler = false
     @prompt_handler = nil
+    
+    # This is used for translating the configuration from 1.3 deployments
     @force_json = true
-    @debug = false
   end
   
   def initialize_copy(source)
@@ -102,14 +110,6 @@ class Properties
     self.props = {}
   end
   
-  def debug(v = nil)
-    if v != nil
-      @debug = v
-    end
-    
-    v
-  end
-  
   def import(properties_obj = {})
     if properties_obj.instance_of?(Hash)
       self.props = properties_obj
@@ -292,11 +292,7 @@ class Properties
   end
   
   # Get the config file value for a property. 
-  def getTemplateValue(key, transform_values_method = nil)
-    if transform_values_method == nil
-      transform_values_method = method(:blank_transform_values_method)
-    end
-    
+  def getTemplateValue(key)
     if key.is_a?(String)
       key_string = key
       key = key.split('.')
@@ -313,7 +309,7 @@ class Properties
         begin
           @in_template_value_prompt_handler[key_string] = true
 
-          value = getPromptHandler().find_template_value(keys, transform_values_method)
+          value = getPromptHandler().find_template_value(keys)
 
           @in_template_value_prompt_handler[key_string] = false
         rescue IgnoreError
@@ -363,10 +359,6 @@ class Properties
     value
   end
   
-  def blank_transform_values_method(matches)
-    ""
-  end
-  
   # Get the property value or return the default if nil
   def getPropertyOr(key, default = "")
     value = getProperty(key)

data/lib/tungsten/script.rb

@@ -28,6 +28,9 @@ module TungstenScript
     # Does this script required to run against an installed Tungsten directory
     @require_installed_directory = true
     
+    # Should unparsed arguments cause an error
+    @allow_unparsed_arguments = false
+    
     # Definition of each command that this script will support
     @command_definitions = {}
     
@@ -44,6 +47,9 @@ module TungstenScript
     # The collected option values from the script input
     @options = {}
     
+    # Parameters loaded from INI files to be parsed
+    @ini_parameters = []
+    
     TU.debug("Begin #{$0} #{ARGV.join(' ')}")
     
     begin
@@ -60,8 +66,11 @@ module TungstenScript
         cleanup(0)
       end
       
+      # Load parameters from the available INI files
+      load_ini_files()
+      # Parse parameters loaded from the INI files and on command line
       parse_options()
-    
+      
       if @options[:autocomplete] == true
         display_autocomplete()
         cleanup(0)
@@ -133,6 +142,15 @@ module TungstenScript
     return @options[option_key]
   end
   
+  # Set the value for option_key if it has not been set
+  def opt_default(option_key, default_value)
+    if opt(option_key) == nil
+      opt(option_key, default_value)
+    end
+    
+    opt(option_key)
+  end
+  
   def add_command(command_key, definition)
     begin
       command_key = command_key.to_sym()
@@ -193,6 +211,69 @@ module TungstenScript
     @option_definitions[option_key][:default] = default
   end
   
+  def load_ini_files
+    # Calculate the INI section name to use
+    section_names = [script_name()]
+    matches = script_name().to_s().match("tungsten_(.*)")
+    if matches && matches.size() > 0
+      script_ini_file = "#{matches[1]}.ini"
+      section_names << matches[1]
+    else
+      script_ini_file = nil
+    end
+    
+    load_ini_parameters("/etc/tungsten/scripts.ini", 
+      section_names)
+    
+    if script_ini_file != nil
+      load_ini_parameters("/etc/tungsten/#{script_ini_file}", 
+        ["__anonymous__"] + section_names)
+    end
+    
+    # Add these arguments to the beginging of the TungstenUtil stack
+    # When the script processes command line options it will read these 
+    # and then be overwritten by and command line options.
+    TU.remaining_arguments = @ini_parameters + TU.remaining_arguments
+  end
+  
+  # Convert the parsed INI contents into the command line argument style
+  def load_ini_parameters(file, section_name)
+    unless File.exists?(file)
+      return
+    end
+    
+    unless section_name.is_a?(Array)
+      section_name = [section_name]
+    end
+    section_name.delete_if{|n| n.to_s() == ""}
+    
+    if section_name.size() == 0
+      return
+    end
+    
+    parameters = TU.parse_ini_file(file, false)
+    section_name.each{
+      |section|
+      if section.to_s() == ""
+        next
+      end
+      
+      unless parameters.has_key?(section)
+        next
+      end
+      
+      parameters[section].each{
+        |line|
+        # Single character parameters get a single dash
+        if line.length() == 1
+          @ini_parameters << "-#{line}"
+        else
+          @ini_parameters << "--#{line}"
+        end
+      }
+    }
+  end
+  
   def parse_options
     opts = OptionParser.new()
     
@@ -269,9 +350,26 @@ module TungstenScript
       end
     end
     
+    unless allow_unparsed_arguments?()
+      unless TU.remaining_arguments.size == 0
+        TU.error("Unable to parse the following arguments: #{TU.remaining_arguments.join(' ')}")
+      end
+    end
+    
     if require_command?() && @command_definitions.size() > 0 && @command == nil
       TU.error("A command was not given for this script. Valid commands are #{@command_definitions.keys().join(', ')} and must be the first argument.")
     end
+    
+    @option_definitions.each{
+      |option_key,definition|
+      
+      if definition[:required] == true
+        if opt(option_key).to_s() == ""
+          arg = definition[:on][0].split(" ")[0]
+          TU.error("Missing value for the #{arg} option")
+        end
+      end
+    }
   end
   
   def script_name
@@ -373,6 +471,14 @@ module TungstenScript
     @require_installed_directory
   end
   
+  def allow_unparsed_arguments?(v = nil)
+    if (v != nil)
+      @allow_unparsed_arguments = v
+    end
+    
+    @allow_unparsed_arguments
+  end
+  
   def require_command?
     true
   end
@@ -394,8 +500,16 @@ module TungstenScript
   end
   
   def cleanup(code = 0)
+    if code != 0
+      log_path = TU.log().path()
+      if log_path.to_s() != "" && File.exist?(log_path)
+        TU.notice("See #{script_log_path()} for more information")
+      end
+    end
+    
     TU.debug("Finish #{$0} #{ARGV.join(' ')}")
     TU.debug("RC: #{code}")
+    
     TU.exit(code)
   end
   
@@ -770,6 +884,14 @@ module MySQLServiceScript
     "mysqldump --defaults-file=#{@options[:my_cnf]} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]} --opt --single-transaction --all-databases --add-drop-database --master-data=2"
   end
   
+  def get_innobackupex_path()
+    path = TU.which("innobackupex-1.5.1")
+    if path.nil?
+      path = TU.which("innobackupex")
+    end
+    return path
+  end
+
   def get_xtrabackup_command
     # Use the configured my.cnf file, or the additional config file 
     # if we created one
@@ -779,7 +901,7 @@ module MySQLServiceScript
       defaults_file = @options[:extra_mysql_defaults_file].path()
     end
     
-    "innobackupex-1.5.1 --defaults-file=#{defaults_file} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]}"
+    "#{get_innobackupex_path()} --defaults-file=#{defaults_file} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]}"
   end
   
   def xtrabackup_supports_argument(arg)