continuent-tools-core 0.0.1

data/lib/tungsten/install.rb

@@ -0,0 +1,332 @@
+class TungstenInstall
+  def initialize(base_path)
+    unless self.class.is_installed?(base_path)
+      raise "Unable to use #{base_path} because it is not an installed Tungsten directory"
+    end
+    
+    @root = TU.cmd_result("cat #{base_path}/.lock")
+    TU.debug("Initialize #{self.class.name} from #{@root}")
+    @settings = {}
+    @topology = nil
+    
+    begin
+      @has_tpm = (TU.cmd_result("#{tpm()} query staging") != "")
+      if @has_tpm == false
+        @has_tpm = (TU.cmd_result("#{tpm()} query dataservices") != "")
+      end
+    rescue
+      @has_tpm = false
+    end
+    
+    # Preload settings about this installation
+    if use_tpm?()
+      # Pull the values from tpm
+      settings([
+        "user",
+        "host_name",
+        HOST_ENABLE_REPLICATOR,
+        HOST_ENABLE_MANAGER,
+        HOST_ENABLE_CONNECTOR,
+        REPL_RMI_PORT,
+        MGR_RMI_PORT,
+        MGR_API,
+        MGR_API_PORT,
+        MGR_API_ADDRESS,
+        "preferred_path"
+        ])
+    else
+      # Read the values from files
+      setting(HOST_ENABLE_REPLICATOR, "true")
+      setting(HOST_ENABLE_MANAGER, "false")
+      setting(HOST_ENABLE_CONNECTOR, "false")
+      setting(REPL_RMI_PORT, TU.cmd_result("grep rmi_port #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/services.properties | grep -v '^#' | awk -F= '{print $2}' | tr -d ' '"))
+      setting("host_name", TU.cmd_result("egrep '^replicator.host=' #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/services.properties | awk -F= '{print $2}'"))
+    end
+  end
+  
+  def self.is_installed?(base_path)
+    if File.exists?("#{base_path}/.manifest") && File.exists?("#{base_path}/.lock")
+      return true
+    else
+      return false
+    end
+  end
+  
+  def root
+    @root
+  end
+  
+  def hostname
+    setting("host_name")
+  end
+  
+  def user
+    # Access the array directly to avoid an infinite loop
+    @settings["user"]
+  end
+  
+  def dataservices
+    ds_list = TU.cmd_result("egrep \"^service.name\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static-* | awk -F \"=\" '{print $2}'").split("\n")
+    
+    if use_tpm?()
+      ds_list = ds_list + TU.cmd_result("#{tpm()} query dataservices | grep COMPOSITE | awk -F \" \" '{print $1}'").split("\n")
+    end
+    
+    ds_list.uniq()
+  end
+  
+  def default_dataservice
+    if is_manager?()
+      setting("dataservice_name")
+    elsif is_replicator?()
+      local_services = TU.cmd_result("egrep -l \"^replicator.service.type=local\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static*").split("\n")
+      if local_services.size() == 0
+        dataservices().get(0)
+      else
+        TU.cmd_result("egrep \"^service.name\" #{local_services[0]} | awk -F \"=\" '{print $2}'")
+      end
+    else
+      dataservices()[0]
+    end
+  end
+  
+  def replication_services
+    TU.cmd_result("egrep \"^service.name\" #{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/conf/static-* | awk -F \"=\" '{print $2}'").split("\n")
+  end
+  
+  def tpm
+    "#{tungsten_sudo_prefix()}#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tools/tpm"
+  end
+  
+  def setting(key, v = nil)
+    if v == nil
+      return settings([key])[key]
+    else
+      @settings[key] = v
+      
+      return v
+    end
+  end
+  
+  def settings(keys)
+    remaining_keys = keys - @settings.keys()
+    if remaining_keys.size() > 0
+      if use_tpm?()    
+        begin
+          JSON.parse(TU.cmd_result("#{tpm()} query values #{remaining_keys.join(' ')}")).each{
+            |k, v|
+            @settings[k] = v
+          }
+        rescue => e
+          TU.exception(e)
+          raise "Unable to load tpm values #{keys.join(' ')}"
+        end
+      else
+        TU.debug("Unable to autodetect settings because tpm was not used to install this directory")
+      end
+    end
+    
+    return_values = {}
+    keys.each{
+      |k|
+      return_values[k] = @settings[k]
+    }
+    return_values
+  end
+
+  def setting_key(first, second, third = nil)
+    if first == CONNECTORS
+      "#{first}.#{TU.to_identifier(hostname())}.#{second}"
+    elsif first == DATASERVICES
+      if third == nil
+        raise "Unable to create setting key for #{first}.#{second}"
+      end
+      "#{first}.#{TU.to_identifier(second)}.#{third}"
+    elsif first == HOSTS
+      "#{first}.#{TU.to_identifier(hostname())}.#{second}"
+    elsif first == MANAGERS
+      if third == nil
+        raise "Unable to create setting key for #{first}.#{second}"
+      end
+      "#{first}.#{TU.to_identifier(second)}_#{TU.to_identifier(hostname())}.#{third}"
+    elsif first == REPL_SERVICES
+      if third == nil
+        raise "Unable to create setting key for #{first}.#{second}"
+      end
+      "#{first}.#{TU.to_identifier(second)}_#{TU.to_identifier(hostname())}.#{third}"
+    else
+      "#{first}.#{TU.to_identifier(hostname())}.#{second}"
+    end
+  end
+  
+  def cctrl
+    "#{tungsten_sudo_prefix()}#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-manager/bin/cctrl -expert -port #{setting(MGR_RMI_PORT)}"
+  end
+  
+  def mgr_api_uri
+    if setting(MGR_API_ADDRESS) == "0.0.0.0"
+      "#{hostname()}:#{setting(MGR_API_PORT)}"
+    else
+      "#{setting(MGR_API_ADDRESS)}:#{setting(MGR_API_PORT)}"
+    end
+  end
+  
+  def status(dataservice = nil)
+    if dataservice == nil
+      dataservice = default_dataservice()
+    end
+    
+    unless dataservices().include?(dataservice)
+      raise "Unable to provide a status for #{dataservice} because it is not defined on this host"
+    end
+    
+    return TungstenStatus.new(self, dataservice)
+  end
+  
+  def topology(dataservice = nil)
+    if dataservice == nil
+      dataservice = default_dataservice()
+    end
+    
+    unless dataservices().include?(dataservice)
+      raise "Unable to provide a topology for #{dataservice} because it is not defined on this host"
+    end
+    
+    return TungstenTopology.new(self, dataservice)
+  end
+  
+  def trepctl(service)
+    "#{tungsten_sudo_prefix()}#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/bin/trepctl -port #{setting(REPL_RMI_PORT)} -service #{service}"
+  end
+  
+  def trepctl_value(service, key)
+    TU.cmd_result("#{trepctl(service)} status | grep #{key} | awk -F: '{print $2}' | tr -d ' '")
+  end
+  
+  def trepctl_property(service, key)
+    properties = JSON.parse(TU.cmd_result("#{trepctl(service)} properties -filter #{key}"))
+    if properties.has_key?(key)
+      return properties[key]
+    else
+      raise "Unable to find a value for #{key} in the output of `trepctl -service #{service} properties`."
+    end
+  end
+  
+  def thl(service)
+    "#{tungsten_sudo_prefix()}#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-replicator/bin/thl -service #{service}"
+  end
+  
+  def service_path(component)
+    "#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-#{component}/bin/#{component}"
+  end
+  
+  def is_running?(component)
+    begin
+      TU.cmd_result("#{service_path(component)} status")
+      return true
+    rescue CommandError
+      return false
+    end
+  end
+  
+  def is_replicator?
+    (setting(HOST_ENABLE_REPLICATOR) == "true")
+  end
+  
+  def is_manager?
+    (setting(HOST_ENABLE_MANAGER) == "true")
+  end
+  
+  def is_connector?
+    (setting(HOST_ENABLE_CONNECTOR) == "true")
+  end
+  
+  def is_commercial?
+    File.exists?("#{@root}/#{CURRENT_RELEASE_DIRECTORY}/tungsten-manager")
+  end
+  
+  def use_tpm?
+    @has_tpm
+  end
+  
+  def ensure_cctrl(cmd, max_tries = 5)
+    i=0
+    while (i<max_tries)
+      begin
+        return TU.cmd_result("echo #{cmd}| #{cctrl}")
+      rescue CommandError
+        TU.debug(e)
+      end
+      i+=1
+    end
+    
+    raise "Unable to execute '#{cmd}' in cctrl"
+  end
+  
+  def inherit_path
+    if setting("preferred_path") != ""
+      ENV['PATH'] = setting("preferred_path").to_s() + ":" + ENV['PATH']
+    end
+  end
+  
+  # Build a sudo prefix to run a command as the tungsten system user
+  def tungsten_sudo_prefix
+    if self.user() == nil || ENV['USER'] == self.user()
+      return ""
+    else
+      return "sudo -u #{self.user()} -n -i "
+    end
+  end
+  
+  def self.get(path)
+    @@instances ||= {}
+    unless @@instances.has_key?(path)
+      @@instances[path] = TungstenInstall.new(path)
+    end
+    return @@instances[path]
+  end
+  
+  class TungstenTopology
+    attr_reader :datasources, :master, :connectors, :dataservices, :type
+
+    def initialize(install, dataservice)
+      @install = install
+      @name = dataservice
+      
+      unless @install.use_tpm?()
+        raise "Unable to parse the topology for #{@name} from #{@install.hostname()}:#{@install.root()} because tpm was not used for installation"
+      end
+      
+      values = @install.settings([
+        "dataservices.#{@name}.dataservice_hosts",
+        "dataservices.#{@name}.dataservice_master_host",
+        "dataservices.#{@name}.dataservice_connectors",
+        "dataservices.#{@name}.dataservice_composite_datasources",
+        "dataservices.#{@name}.dataservice_topology"
+      ])
+      
+      @type = values["dataservices.#{@name}.dataservice_topology"]
+      @members = values["dataservices.#{@name}.dataservice_hosts"].to_s().split(",")
+      @master = values["dataservices.#{@name}.dataservice_master_host"]
+      @connectors = values["dataservices.#{@name}.dataservice_connectors"].to_s().split(",")
+      @dataservices = values["dataservices.#{@name}.dataservice_composite_datasources"].to_s().split(",")
+    end
+    
+    def is_composite?
+      (@dataservices.size() > 0)
+    end
+    
+    def to_hash
+      {
+        :hostname => @install.hostname(),
+        :root => @install.root(),
+        :is_composite => is_composite?(),
+        :type => @type,
+        :members => @members,
+        :master => @master,
+        :connectors => @connectors,
+        :dataservices => @dataservices
+      }
+    end
+  end
+end

data/lib/tungsten/properties.rb

@@ -0,0 +1,476 @@
+#
+# TUNGSTEN SCALE-OUT STACK
+# Copyright (C) 2009 Continuent, Inc.
+# All rights reserved
+#
+
+require "date"
+
+# Defines a properties object. 
+class Properties
+  attr_accessor :props,:use_prompt_handler,:force_json
+  
+  # Initialize with some base values. 
+  def initialize
+    @props = {}
+    @in_prompt_handler = {}
+    @in_template_value_prompt_handler = {}
+    @use_prompt_handler = false
+    @prompt_handler = nil
+    @force_json = true
+    @debug = false
+  end
+  
+  def initialize_copy(source)
+    super(source)
+    @props = Marshal::load(Marshal::dump(@props))
+    @in_prompt_handler = {}
+    @in_template_value_prompt_handler = {}
+    @use_prompt_handler = source.use_prompt_handler
+    @prompt_handler = nil
+    @force_json = source.force_json
+  end
+  
+  # Read properties from a file. 
+  def load(properties_filename)
+    file_contents = ""
+    
+    File.open(properties_filename, 'r') do |file|
+      file.read.each_line do |line|
+        line.strip!
+        unless (line =~ /^#.*/)
+          file_contents = file_contents + line
+        end
+      end
+      
+      begin
+        parsed_contents = JSON.parse(file_contents)
+      rescue Exception => e
+        if file_contents == ""
+          parsed_contents = {}
+        elsif file_contents[0,1] == "{"
+          raise "There was an error parsing the config file: #{e.message}"
+        end
+      end
+      
+      if parsed_contents && parsed_contents.instance_of?(Hash)
+        @props = parsed_contents
+      elsif @force_json == true
+        raise "There was an error parsing the JSON config file: #{properties_filename}.  Try using the migration procedure if you have an old configuration file."
+      else
+        new_props = {}
+        
+        file.rewind()
+        file.read.each_line do |line|
+          line.strip!
+          
+          if (line =~ /^([\w\.]+)\[?([\w\.]+)?\]?\s*=\s*(\S.*)/)
+            key = $1
+            value = $3
+        
+            if $2
+              new_props[key] = {} unless new_props[key]
+              new_props[key][$2] = value
+            else
+              new_props[key] = value
+            end
+          elsif (line =~ /^([\w\.]+)\s*=/)
+            key = $1
+            value = ""
+            new_props[key] = value
+          end
+        end
+        
+        @props = new_props
+      end
+      
+      original_props = @props.dup
+      
+      if original_props != @props
+        Configurator.instance.warning("Deprecated keys in the config file were updated")
+      end
+    end
+  end
+  
+  # Read properties from a file. 
+  def load_and_initialize(properties_filename, keys_module)
+    load(properties_filename)
+    init(keys_module)
+  end
+  
+  def reset
+    self.props = {}
+  end
+  
+  def debug(v = nil)
+    if v != nil
+      @debug = v
+    end
+    
+    v
+  end
+  
+  def import(properties_obj = {})
+    if properties_obj.instance_of?(Hash)
+      self.props = properties_obj
+    elsif properties_obj.instance_of?(Properties)
+      self.props = properties_obj.props
+    else
+      raise "You must pass in a Hash or Properties object to import"
+    end
+  end
+  
+  # Write properties to a file.  We use signal protection to avoid getting
+  # interrupted half-way through. 
+  def store(properties_filename, use_json = true)
+    # Protect I/O with trap for Ctrl-C. 
+    interrupted = false
+    old_trap = trap("INT") {
+      interrupted = true;
+    }
+    
+    # Write. 
+    File.open(properties_filename, 'w') do |file|
+      file.printf "# Tungsten configuration properties\n"
+      file.printf "# Date: %s\n", DateTime.now
+      
+      if use_json == false
+        @props.sort.each do | key, value |
+          file.printf "%s=%s\n", key, value
+        end
+      else
+        file.print self.to_s
+      end
+    end
+    
+    # Check for interrupt and restore handler. 
+    if (interrupted) 
+      puts
+      puts ("Configuration interrupted") 
+      exit 1;
+    else
+      trap("INT", old_trap);
+    end
+  end
+  
+  # Return the size of the properties object. 
+  def size()
+    @props.size
+  end
+  
+  def to_s
+    JSON.pretty_generate(@props)
+  end
+  
+  def output()
+    TU.output(self.to_s)
+  end
+  
+  def force_output()
+    TU.force_output(self.to_s)
+  end
+  
+  # Fetch a nested hash value
+  def getNestedProperty(attrs)
+    if attrs.is_a?(String)
+      attrs = attrs.split('.')
+    end
+    
+    attr_count = attrs.size
+    current_val = @props
+    for i in 0..(attr_count-1)
+      attr_name = attrs[i]
+      return current_val[attr_name] if i == (attr_count-1)
+      return nil if current_val[attr_name].nil?
+      current_val = current_val[attr_name]
+    end
+    
+    return nil
+  end
+  
+  def setNestedProperty(new_val, attrs)
+    attr_count = attrs.size
+    current_val = @props
+    for i in 0..(attr_count-1)
+      attr_name = attrs[i]
+      if i == (attr_count-1)
+        return setHashProperty(current_val, attr_name, new_val)
+      end
+      current_val[attr_name] = {} if current_val[attr_name].nil?
+      current_val = current_val[attr_name]
+    end
+  end
+  
+  def setHashProperty(hash, key, value)
+    if value == nil  || value == []
+      return (hash.delete(key))
+    else
+      if value.is_a?(Hash)
+        hash[key] ||= {}
+        value.each{|sub_key,sub_value|
+          setHashProperty(hash[key], sub_key, sub_value)
+        }
+      else
+        return (hash[key] = value)
+      end
+    end
+  end
+  
+  # Get a property value. 
+  def getProperty(key, allow_disabled = false)
+    if key.is_a?(String)
+      key_string = key
+      key = key.split('.')
+    else
+      key_string = key.join('.')
+    end
+    
+    value = getNestedProperty(key)
+    if value != nil
+      return value
+    end
+    
+    if usePromptHandler()
+      findProperty = lambda do |keys|
+        if @in_prompt_handler[key_string] == true
+          return nil
+        end
+
+        begin
+          @in_prompt_handler[key_string] = true
+
+          value = getPromptHandler().get_property(keys, allow_disabled)
+
+          @in_prompt_handler[key_string] = false
+        rescue IgnoreError
+          @in_prompt_handler[key_string] = false
+        rescue => e
+          @in_prompt_handler[key_string] = false
+          raise e
+        end
+
+        return value
+      end
+    
+    
+      value = findProperty.call(key)
+    else
+      findProperty = lambda do |keys|
+        return getNestedProperty(keys)
+      end
+    end
+    
+    if value == nil && key.size == 1 && (host = getNestedProperty([DEPLOYMENT_HOST]))
+      value = findProperty.call([HOSTS, host, key[0]])
+      
+      if value == nil && key.size == 1
+        dataservice = getProperty(DEPLOYMENT_DATASERVICE)
+        
+        if dataservice
+          value = findProperty.call([DATASERVICES, dataservice, key[0]])
+        
+          if value == nil
+            value = findProperty.call([REPL_SERVICES, dataservice + "_" + host, key[0]])
+          end
+        
+          if value == nil
+            value = findProperty.call([MANAGERS, dataservice + "_" + host, key[0]])
+          end
+        end
+      end
+      
+      if value == nil
+        value = findProperty.call([CONNECTORS, host, key[0]])
+      end
+      
+      if value == nil && key.size == 1 && (svc = getNestedProperty([DEPLOYMENT_SERVICE]))
+        value = findProperty.call([REPL_SERVICES, svc, key[0]])
+      end
+    end
+    
+    value
+  end
+  
+  # Get the config file value for a property. 
+  def getTemplateValue(key, transform_values_method = nil)
+    if transform_values_method == nil
+      transform_values_method = method(:blank_transform_values_method)
+    end
+    
+    if key.is_a?(String)
+      key_string = key
+      key = key.split('.')
+    else
+      key_string = key.join('.')
+    end
+    
+    if usePromptHandler()
+      findProperty = lambda do |keys|
+        if @in_template_value_prompt_handler[key_string] == true
+          return nil
+        end
+
+        begin
+          @in_template_value_prompt_handler[key_string] = true
+
+          value = getPromptHandler().find_template_value(keys, transform_values_method)
+
+          @in_template_value_prompt_handler[key_string] = false
+        rescue IgnoreError
+          @in_template_value_prompt_handler[key_string] = false
+        rescue => e
+          @in_template_value_prompt_handler[key_string] = false
+          raise e
+        end
+
+        return value
+      end
+    
+      value = findProperty.call(key)
+    end
+    
+    if value == nil && key.size == 1 && (host = getNestedProperty([DEPLOYMENT_HOST]))
+      value = findProperty.call([HOSTS, host, key[0]])
+      
+      if value == nil && key.size == 1
+        dataservice = getNestedProperty([DEPLOYMENT_DATASERVICE])
+        if dataservice == nil
+          dataservice = getProperty(DEPLOYMENT_DATASERVICE)
+        end
+        
+        if dataservice
+          value = findProperty.call([DATASERVICES, dataservice, key[0]])
+        
+          if value == nil
+            value = findProperty.call([REPL_SERVICES, dataservice + "_" + host, key[0]])
+          end
+        
+          if value == nil
+            value = findProperty.call([MANAGERS, dataservice + "_" + host, key[0]])
+          end
+        end
+      end
+      
+      if value == nil
+        value = findProperty.call([CONNECTORS, host, key[0]])
+      end
+    end
+    
+    if value == nil && key.size == 1 && (svc = getNestedProperty([DEPLOYMENT_SERVICE]))
+      value = findProperty.call([REPL_SERVICES, svc, key[0]])
+    end
+    
+    value
+  end
+  
+  def blank_transform_values_method(matches)
+    ""
+  end
+  
+  # Get the property value or return the default if nil
+  def getPropertyOr(key, default = "")
+    value = getProperty(key)
+    if value == nil
+      default
+    else
+      value
+    end
+  end
+  
+  def getNestedPropertyOr(key, default = "")
+    value = getNestedProperty(key)
+    if value == nil
+      default
+    else
+      value
+    end
+  end
+  
+  # Set a property value. 
+  def setProperty(key, value)
+    if key.is_a?(String)
+      key = key.split('.')
+    end
+    
+    setNestedProperty(value, key)
+  end 
+  
+  # Set the property to a value only if it is currently unset. 
+  def setDefault(key, value)
+    if key.is_a?(String)
+      key = key.split('.')
+    end
+    
+    if getNestedProperty(key) == nil
+      setNestedProperty(value, key)
+    end
+  end
+  
+  # Set multiple properties from a delimited string of key value pairs. 
+  def setPropertiesFromList(list, delimiter)
+    keyValuePairs = list.split(delimiter)
+    keyValuePairs.each do |pair|
+      if pair =~ /^(.*)=(.*)$/
+        key = $1
+        value = $2
+        setProperty(key, value)
+      end
+    end
+  end
+  
+  # Get the underlying hash table. 
+  def hash()
+    @props
+  end
+  
+  def usePromptHandler
+    return @use_prompt_handler
+  end
+  
+  def getPromptHandler
+    unless usePromptHandler()
+      return nil
+    end
+    
+    unless @prompt_handler
+      @prompt_handler = ConfigurePromptHandler.new(self)
+    end
+    
+    return @prompt_handler
+  end
+  
+  def empty?
+    (@props.size() == 0)
+  end
+  
+  def override(key, value = {})
+    if value == nil || value == ""
+      value = {}
+    end
+    
+    setProperty(key, getNestedPropertyOr(key, {}).merge(value))
+  end
+  
+  def include(key, value = {})
+    if value == nil || value == ""
+      value = {}
+    end
+    
+    setProperty(key, value.merge(getNestedPropertyOr(key, {})))
+  end
+  
+  def append(key, value = [])
+    if value == nil || value == ""
+      value = []
+    end
+    
+    if ! value.kind_of?(Array)
+       value=Array(value) 
+    end
+    currentvalue=getNestedPropertyOr(key, [])
+    if ! currentvalue.kind_of?(Array)
+        currentvalue=Array(currentvalue)
+    end
+    setProperty(key, (currentvalue+value).uniq())
+  end
+end