continuent-tools-core 0.9.0 → 0.10.6

data/lib/tungsten/datasources/mysql.rb

@@ -17,7 +17,8 @@ class TungstenScriptMySQLDatasource < TungstenScriptDatasource
     end
     
     begin
-      TU.cmd_result("#{@ti.sudo_prefix()}#{get_service_command()} stop")
+      stop_command = @ti.setting(TI.setting_key(REPL_SERVICES, @service, "repl_datasource_service_stop"))
+      TU.cmd_result("#{@ti.sudo_prefix()}#{stop_command}")
     rescue CommandError
     end
     
@@ -51,7 +52,8 @@ class TungstenScriptMySQLDatasource < TungstenScriptDatasource
   
   def _start_server
     begin
-      TU.cmd_result("#{@ti.sudo_prefix()}#{get_service_command()} start")
+      start_command = @ti.setting(TI.setting_key(REPL_SERVICES, @service, "repl_datasource_service_start"))
+      TU.cmd_result("#{@ti.sudo_prefix()}#{start_command}")
     rescue CommandError
     end
     
@@ -94,40 +96,6 @@ class TungstenScriptMySQLDatasource < TungstenScriptDatasource
     end
   end
   
-  def get_service_command
-    if @mysql_service_command == nil
-      if @mysql_service_command == nil
-        @mysql_service_command = @ti.setting(TI.setting_key(REPL_SERVICES, @service, "repl_datasource_boot_script"))
-      end
-      if @mysql_service_command == nil
-        begin
-          service_command=TU.cmd_result("which service")
-          if TU.cmd("#{@ti.sudo_prefix()}test -x #{service_command}")
-            if TU.cmd("#{@ti.sudo_prefix()}test -x /etc/init.d/mysqld")
-              @mysql_service_command = "#{service_command} mysqld"
-            elsif TU.cmd("#{@ti.sudo_prefix()}test -x /etc/init.d/mysql")
-              @mysql_service_command = "#{service_command} mysql"
-            else
-              TU.error "Unable to determine the service command to start/stop mysql"
-            end
-          else
-            if TU.cmd("#{@ti.sudo_prefix()}test -x /etc/init.d/mysqld")
-              @mysql_service_command = "/etc/init.d/mysqld"
-            elsif TU.cmd("#{@ti.sudo_prefix()}test -x /etc/init.d/mysql")
-              @mysql_service_command = "/etc/init.d/mysql"
-            else
-              TU.error "Unable to determine the service command to start/stop mysql"
-            end
-          end
-        rescue CommandError
-          TU.error "Unable to determine the service command to start/stop mysql"
-        end
-      end
-    end
-    
-    @mysql_service_command
-  end
-  
   def get_system_user
     if @mysql_user == nil
       @mysql_user = get_option("user")

data/lib/tungsten/exec.rb

@@ -1,6 +1,6 @@
 require "open4"
 require "resolv"
-require "ifconfig"
+require "ipparse"
 
 class TungstenUtil
   def initialize()
@@ -283,16 +283,14 @@ class TungstenUtil
       return cmd_result(command)
     end
 
-    self.synchronize() {
-      unless defined?(Net::SSH)
-        begin
-          require "openssl"
-        rescue LoadError
-          raise("Unable to find the Ruby openssl library. Try installing the openssl package for your version of Ruby (libopenssl-ruby#{RUBY_VERSION[0,3]}).")
-        end
-        require 'net/ssh'
+    unless defined?(Net::SSH)
+      begin
+        require "openssl"
+      rescue LoadError
+        raise("Unable to find the Ruby openssl library. Try installing the openssl package for your version of Ruby (libopenssl-ruby#{RUBY_VERSION[0,3]}).")
       end
-    }
+      require 'net/ssh'
+    end
     
     ssh_user = get_ssh_user(user)
     if user != ssh_user
@@ -470,15 +468,15 @@ class TungstenUtil
     end
 
     debug("Search ifconfig for #{ip_addresses.join(', ')}")
-    ifconfig = IfconfigWrapper.new().parse()
-    ifconfig.each{
-      |iface|
+    ipparsed = IPParse.new().get_interfaces()
+    ipparsed.each{
+      |iface, addresses|
 
       begin
         # Do a string comparison so that we only match the address portion
-        iface.addresses().each{
-          |a|
-          if ip_addresses.include?(a.to_s())
+        addresses.each{
+          |type, details|
+          if ip_addresses.include?(details[:address])
             return true
           end
         }
@@ -498,6 +496,7 @@ class TungstenUtil
       ip_addresses = Timeout.timeout(5) {
         Resolv.getaddresses(hostname)
       }
+      ip_addresses.delete_if{|ip| ip.to_s() == ""}
       
       if ip_addresses.length == 0
         begin
@@ -528,6 +527,7 @@ class TungstenUtil
       ip_addresses = Timeout.timeout(5) {
         Resolv.getaddresses(hostname)
       }
+      ip_addresses.delete_if{|ip| ip.to_s() == ""}
 
       if ip_addresses.length == 0
         begin

data/lib/tungsten/script.rb

@@ -614,9 +614,10 @@ module OfflineServicesScript
     })
     
     add_option(:offline, {
-      :on => "--offline",
+      :on => "--offline String",
       :help => "Put required replication services offline before processing",
-      :default => false
+      :default => false,
+      :parse => method(:parse_boolean_option_blank_is_true)
     })
     
     add_option(:offline_timeout, {
@@ -627,9 +628,10 @@ module OfflineServicesScript
     })
     
     add_option(:online, {
-      :on => "--online",
+      :on => "--online String",
       :help => "Put required replication services online after successful processing",
-      :default => false
+      :default => false,
+      :parse => method(:parse_boolean_option_blank_is_true)
     })
   end
   
@@ -671,21 +673,32 @@ module OfflineServicesScript
               ds_list.each{
                 |ds|
                 threads << Thread.new{
+                  use_manager = false
+                  
                   if TI.is_manager?()
-                    begin
-                      status = TI.status(ds)
-                      unless status.is_replication?() == true
-                        get_manager_api.call("#{ds}/#{TI.hostname()}", 'shun')
-                        TU.cmd_result("#{TI.trepctl(ds)} offline")
-                      else
-                        TU.cmd_result("#{TI.trepctl(ds)} offline")
+                    status = TI.status(ds)
+                    
+                    # Make sure this is actually a physical dataservice
+                    if status.is_physical?()
+                      # Does this datasource actually appear in the status
+                      # It may not if the host hasn't been provisioned
+                      if status.datasources().index(TI.hostname()) != nil
+                        use_manager = true
                       end
-                    rescue => e
-                      TU.exception(e)
-                      raise("Unable to put replication services offline")
                     end
-                  else
+                  end
+                  
+                  begin
+                    if use_manager == true
+                      get_manager_api.call("#{ds}/#{TI.hostname()}", 'shun')
+                    end
+                  
+                    # The trepctl offline command is required even when using 
+                    # the manager because shun doesn't affect the replicator
                     TU.cmd_result("#{TI.trepctl(ds)} offline")
+                  rescue => e
+                    TU.exception(e)
+                    raise("Unable to put replication services offline")
                   end
                 }
               }
@@ -751,23 +764,36 @@ module OfflineServicesScript
             end
             
             if online == true
+              use_manager = false
+              
               if TI.is_manager?()
-                begin
-                  status = TI.status(ds)
-                  unless status.is_replication?() == true
-                    TU.cmd_result("echo 'datasource #{TI.hostname()} recover' | #{TI.cctrl()}")
-                    #get_manager_api.call("#{ds}/#{TI.hostname()}", 'recover')
-                  else
-                    TU.cmd_result("#{TI.trepctl(ds)} online")
+                status = TI.status(ds)
+                
+                # Make sure this is actually a physical dataservice
+                if status.is_physical?()
+                  # Does this datasource actually appear in the status
+                  # It may not if the host hasn't been provisioned
+                  if status.datasources().index(TI.hostname()) != nil
+                    use_manager = true
                   end
-                rescue => e
-                  TU.exception(e)
-                  raise("The #{ds} replication service did not come online")
                 end
-              else
-                TU.cmd_result("#{TI.trepctl(ds)} online")
               end
               
+              begin
+                if use_manager == true
+                  # Bring the replicator and the datasource ONLINE
+                  TU.cmd_result("echo 'datasource #{TI.hostname()} recover' | #{TI.cctrl()}")
+                else
+                  # Bring just the replicator ONLINE
+                  TU.cmd_result("#{TI.trepctl(ds)} online")
+                end
+              rescue => e
+                TU.exception(e)
+                raise("The #{ds} replication service did not come online")
+              end
+              
+              # Verify the replicator is in fact ONLINE since the recover 
+              # command may have not returned the right error
               unless TI.trepctl_value(ds, "state") == "ONLINE"
                 raise("Unable to put the #{ds} replication service online")
               end
@@ -871,34 +897,6 @@ module MySQLServiceScript
       if @options[:mysqluser].to_s() == ""
         @options[:mysqluser] = "mysql"
       end
-    
-      if @options[:mysql_service_command] == nil
-        @options[:mysql_service_command] = TI.setting(TI.setting_key(REPL_SERVICES, @options[:service], "repl_datasource_boot_script"))
-      end
-      if @options[:mysql_service_command] == nil
-        begin
-          service_command=TU.cmd_result("which service")
-          if TU.cmd("#{sudo_prefix()}test -x #{service_command}")
-            if TU.cmd("#{sudo_prefix()}test -x /etc/init.d/mysqld")
-              @options[:mysql_service_command] = "#{service_command} mysqld"
-            elsif TU.cmd("#{sudo_prefix()}test -x /etc/init.d/mysql")
-              @options[:mysql_service_command] = "#{service_command} mysql"
-            else
-              TU.error "Unable to determine the service command to start/stop mysql"
-            end
-          else
-            if TU.cmd("#{sudo_prefix()}test -x /etc/init.d/mysqld")
-              @options[:mysql_service_command] = "/etc/init.d/mysqld"
-            elsif TU.cmd("#{sudo_prefix()}test -x /etc/init.d/mysql")
-              @options[:mysql_service_command] = "/etc/init.d/mysql"
-            else
-              TU.error "Unable to determine the service command to start/stop mysql"
-            end
-          end
-        rescue CommandError
-          TU.error "Unable to determine the service command to start/stop mysql"
-        end
-      end
     end
   end
   
@@ -1019,76 +1017,14 @@ module MySQLServiceScript
   end
   
   def start_mysql_server
-    TU.notice("Start the MySQL service")
-    begin
-      TU.cmd_result("#{sudo_prefix()}#{@options[:mysql_service_command]} start")
-    rescue CommandError
-    end
-    
-    # Wait 30 seconds for the MySQL service to be responsive
-    begin
-      Timeout.timeout(30) {
-        while true
-          begin
-            if get_mysql_result("SELECT 1") != nil
-              break
-            end
-            
-            # Pause for a second before running again
-            sleep 1
-          rescue
-          end
-        end
-      }
-    rescue Timeout::Error
-      raise "The MySQL server has taken too long to start"
-    end
+    ds = TI.datasource(@options[:service])
+    ds.start()
   end
   
   # Make sure that the mysql server is stopped by stopping it and checking
   # the process has disappeared
   def stop_mysql_server
-    TU.notice("Stop the MySQL service")
-    begin
-      pid_file = get_mysql_variable("pid_file")
-      pid = TU.cmd_result("#{sudo_prefix()}cat #{pid_file}")
-    rescue CommandError
-      pid = ""
-    end
-    
-    begin
-      TU.cmd_result("#{sudo_prefix()}#{@options[:mysql_service_command]} stop")
-    rescue CommandError
-    end
-    
-    begin
-      # Verify that the stop command worked properly
-      # We are expecting an error so we have to catch the exception
-      TU.cmd_result("#{get_mysql_command()} -e \"select 1\" > /dev/null 2>&1")
-      raise "Unable to properly shutdown the MySQL service"
-    rescue CommandError
-    end
-    
-    # We saw issues where MySQL would not close completely. This will
-    # watch the PID and make sure it does not appear
-    unless pid.to_s() == ""
-      begin
-        TU.debug("Verify that the MySQL pid has gone away")
-        Timeout.timeout(30) {
-          pid_missing = false
-          
-          while pid_missing == false do
-            begin
-              TU.cmd_result("#{sudo_prefix()}ps -p #{pid}")
-              sleep 5
-            rescue CommandError
-              pid_missing = true
-            end
-          end
-        }
-      rescue Timeout::Error
-        raise "Unable to verify that MySQL has fully shutdown"
-      end
-    end
+    ds = TI.datasource(@options[:service])
+    ds.stop()
   end
 end

data/lib/tungsten/util.rb

@@ -14,7 +14,6 @@ class TungstenUtil
     @force = false
     @json_interface = false
     @json_message_cache = []
-    @mutex = Mutex.new
     
     # Create a temporary file to hold log contents
     @log = Tempfile.new("tlog")
@@ -65,8 +64,6 @@ class TungstenUtil
                                         
                                           if val_parts[0] == "timeout"
                                             val_parts[1] = val_parts[1].to_i
-                                          elsif val_parts[0] == "verbose"
-                                            val_parts[1] = val_parts[1].to_sym
                                           end
 
                                           @ssh_options[val_parts[0].to_sym] = val_parts[1]
@@ -629,10 +626,4 @@ class TungstenUtil
     
     return hash
   end
-  
-  def synchronize(&block)
-    @mutex.synchronize do
-      block.call()
-    end
-  end
 end

data/providers/awsec2.rb

@@ -0,0 +1,141 @@
+class AWSEC2TungstenDirectoryProvider < TungstenDirectoryProvider
+  DEFAULT_HOSTNAME_TAG = "Name"
+  
+  def self.get_regex()
+    "aws\.ec2.*"
+  end
+  
+  def get_entries(aws_info)
+    hostname_tag = DEFAULT_HOSTNAME_TAG
+    require_hostname_tag = false
+    aws_entries = {}
+    
+    if aws_info.has_key?("hostname_tag")
+      hostname_tag = aws_info["hostname_tag"]
+      
+      if aws_info.has_key?("require_hostname_tag") && aws_info["require_hostname_tag"].to_s() == "false"
+        if aws_info["hostname_tag"] == DEFAULT_HOSTNAME_TAG
+          # Only allow the use of DEFAULT_HOSTNAME_TAG
+          require_hostname_tag = true
+        else
+          # Allow the use of the given tag and DEFAULT_HOSTNAME_TAG
+          require_hostname_tag = false
+        end
+      else
+        # Only allow the use of DEFAULT_HOSTNAME_TAG
+        require_hostname_tag = true
+      end
+    end
+    
+    unless defined?(AWS)
+      begin
+        gem 'aws-sdk', '>=1.14.0', '<2.0.0'
+        require 'aws/ec2'
+      rescue LoadError
+        raise "The aws-sdk Ruby gem or rubygem-aws-sdk package is required for this class"
+      end
+    end
+    AWS.eager_autoload!(AWS::EC2)
+    
+    if aws_info.has_key?("access_key_id")
+      AWS.config({
+        :access_key_id => aws_info["access_key_id"]
+      })
+    end
+    if aws_info.has_key?("secret_access_key")
+      AWS.config({
+        :secret_access_key => aws_info["secret_access_key"]
+      })
+    end
+    if aws_info.has_key?("proxy_uri")
+      AWS.config({
+        :proxy_uri => aws_info["proxy_uri"]
+      })
+    end
+    ec2 = AWS::EC2.new()
+
+    region_index = -1
+    region_threads = []
+    region_results = []
+    
+    if aws_info.has_key?("regions")
+      regions = aws_info["regions"]
+      if regions.is_a?(String)
+        regions = regions.split(",")
+      end
+    else
+      regions = ec2.regions.map(&:name)
+    end
+    regions.each{|r|
+      region_threads << Thread.new{
+        index = Thread.exclusive{ (region_index = region_index+1) }
+        region = regions[index]
+        region_results[index] = {}
+        TU.debug("Collect ec2_hosts from #{region}")
+        
+        begin
+          region_ec2 = AWS::EC2.new(:region => region)
+          instances = region_ec2.instances().filter('instance-state-name', "running")
+
+          if aws_info.has_key?("tag_key")
+            if aws_info.has_key?("tag_value")
+              instances = instances.with_tag(aws_info["tag_key"], aws_info["tag_value"])
+            else
+              instances = instances.tagged(aws_info["tag_key"])
+            end
+          else
+            instances = instances.tagged('tungsten-ServerType')
+          end
+          instances.each{
+            |ins|
+            tags = ins.tags.to_h()
+          
+            name = tags.to_h()[hostname_tag]
+            
+            if name.to_s() == "" && require_hostname_tag == false
+              # Allow the DEFAULT_HOSTNAME_TAG to be used as a backup
+              name = tags.to_h()[DEFAULT_HOSTNAME_TAG]
+            end
+            
+            if name.to_s() == ""
+              TU.error("Unable to identify the hostname for #{ins.id} in #{region}")
+            end
+          
+          
+            if ins.vpc_id != nil
+              location = ins.availability_zone + "." + ins.vpc_id
+            else
+              location = ins.availability_zone
+            end
+          
+            TU.debug("Found #{ins.id}")
+            region_results[index][name] = {
+              'id' => ins.id.to_s(),
+              'hostname' => name,
+              'location' => location,
+              'public-address' => ins.public_ip_address,
+              'private-address' => ins.private_ip_address,
+              'tags' => tags.to_h(),
+              'provider' => "aws.ec2",
+              'autodetect-key' => @key
+            }
+          }
+        rescue AWS::EC2::Errors::AuthFailure => af
+          TU.debug("Unable to find instances in #{region}: #{af.message}. Operation will continue.")
+        rescue => e
+          TU.debug("Error finding instances in #{region}: #{e.message}")
+          raise e
+        end
+      }
+    }
+    
+    region_threads.each{|t| t.join() }
+
+    region_results.each{
+      |region_result|
+      aws_entries.merge!(region_result)
+    }
+    
+    aws_entries
+  end
+end