continuent-tools-core 0.9.0 → 0.10.6

data/bin/tungsten_migrate_schema

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2014 Continuent, Inc.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.  You may obtain
+# a copy of the License at
+# 
+#         http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Initial developer(s): Jeff Mace
+# Contributor(s):
+
+begin
+  require 'rubygems'
+  gem 'continuent-tools-core'
+rescue LoadError
+end
+
+require 'continuent-tools-core'
+require 'tempfile'
+
+class TungstenMigrateSchema
+  include TungstenScript
+  include MySQLServiceScript
+  include OfflineSingleServiceScript
+  private
+  
+  def main
+    f = Tempfile.new("#{script_name()}")
+    
+    TU.debug("Create mysqldump in #{f.path()}")
+    if opt(:log_updates) == false
+      TU.cmd_result("echo \"SET SESSION SQL_LOG_BIN=0;\n\" >> #{f.path()}")
+    end
+    
+    if opt(:drop_original_schema) == true
+      TU.cmd_result("echo \"DROP SCHEMA IF EXISTS \\`#{opt(:to)}\\`;\n\" >> #{f.path()}")
+    end
+    
+    TU.cmd_result("echo \"CREATE SCHEMA \\`#{opt(:to)}\\`;\n\" >> #{f.path()}")
+    TU.cmd_result("echo \"USE '#{opt(:to)}';\n\" >> #{f.path()}")
+    TU.cmd_result("#{get_mysqldump_command(opt(:from))} >> #{f.path()}")
+    
+    if opt(:drop_target_schema) == true
+      TU.cmd_result("echo \"DROP SCHEMA \\`#{opt(:from)}\\`;\n\" >> #{f.path()}")
+    end
+    
+    TU.cmd_result("cat #{f.path()} | #{get_mysql_command()}")
+  end
+  
+  def get_mysqldump_command(schema)
+    "mysqldump --defaults-file=#{@options[:my_cnf]} --host=#{@options[:mysqlhost]} --port=#{@options[:mysqlport]} --opt --single-transaction #{schema}"
+  end
+  
+  def configure
+    super()
+    
+    add_option(:from, {
+      :on => "--from-schema String",
+      :help => "The existing schema name to use",
+      :required => true,
+    })
+    
+    add_option(:to, {
+      :on => "--to-schema String",
+      :help => "The schema name to create",
+      :required => true,
+    })
+    
+    add_option(:log_updates, {
+      :on => "--log-updates String",
+      :parse => method(:parse_boolean_option),
+      :default => false,
+      :help => "Force the script to log updates to the database",
+    })
+    
+    add_option(:drop_original_schema, {
+      :on => "--drop-original-schema String",
+      :parse => method(:parse_boolean_option),
+      :default => false,
+      :help => "Drop the schema identified by --from-schema",
+    })
+    
+    add_option(:drop_target_schema, {
+      :on => "--drop-target-schema String",
+      :parse => method(:parse_boolean_option),
+      :default => false,
+      :help => "Drop the schema identified by --to-schema",
+    })
+  end
+  
+  def validate
+    super()
+    
+    unless TU.is_valid?()
+      return TU.is_valid?()
+    end
+    
+    unless opt(:drop_original_schema) == true
+      if get_mysql_result("SHOW SCHEMAS LIKE '#{opt(:from)}'") == ""
+        TU.error("Unable to find an existing '#{opt(:from)}' schema")
+      end
+    end
+    
+    unless opt(:drop_target_schema) == true
+      if get_mysql_result("SHOW SCHEMAS LIKE '#{opt(:to)}'") != ""
+        TU.error("There is already an existing '#{opt(:to)}' schema")
+      end
+    end
+  end
+  
+  def script_name
+    "tungsten_migrate_schema"
+  end
+  
+  self.new().run()
+end

data/lib/continuent-tools-core.rb

@@ -20,6 +20,22 @@
 require 'tungsten'
 
 class TungstenUtil
+  # Override the method imported from tpm to include '>>' to avoid detecting
+  # an error that wasn't thrown by TungstenScript
+  def parse_log_level(line)
+    prefix = line[0,8]
+    
+    case prefix.strip
+    when "ERROR >>" then Logger::ERROR
+    when "WARN  >>" then Logger::WARN
+    when "DEBUG >>" then Logger::DEBUG
+    when "NOTE  >>" then Logger::NOTICE
+    when "INFO  >>" then Logger::INFO
+    else
+      nil
+    end
+  end
+  
   # A wrapper for running another Tungsten script. This will automatically
   # forward messages to the console and add any TungstenScript options to 
   # the command.
@@ -139,6 +155,113 @@ class TungstenUtil
 
     return result
   end
+  
+  def scp_result(local_file, remote_file, host, user)
+    unless File.file?(local_file)
+      debug("Unable to copy '#{local_file}' because it doesn't exist")
+      raise MessageError.new("Unable to copy '#{local_file}' because it doesn't exist")
+    end
+
+    if is_localhost?(host) && 
+        user == whoami()
+      debug("Copy #{local_file} to #{remote_file}")
+      return FileUtils.cp(local_file, remote_file)
+    end
+
+    self.synchronize() {
+      unless defined?(Net::SCP)
+        begin
+          require "openssl"
+        rescue LoadError
+          raise("Unable to find the Ruby openssl library. Try installing the openssl package for your version of Ruby (libopenssl-ruby#{RUBY_VERSION[0,3]}).")
+        end
+        require 'net/scp'
+      end
+    }
+
+    ssh_user = get_ssh_user(user)
+    if user != ssh_user
+      debug("SCP user changed to #{ssh_user}")
+    end
+
+    connection_error = "Net::SCP was unable to copy #{local_file} to #{host}:#{remote_file} as #{ssh_user}.  Check that #{host} is online, #{ssh_user} exists and your SSH private keyfile or ssh-agent settings. Try adding --net-ssh-option=port=<SSH port number> if you are using an SSH port other than 22.  Review http://docs.continuent.com/helpwithsshandtpm for more help on diagnosing SSH problems."
+    debug("Copy #{local_file} to #{host}:#{remote_file} as #{ssh_user}")
+    begin
+      Net::SCP.start(host, ssh_user, get_ssh_options) do |scp|
+        scp.upload!(local_file, remote_file, get_ssh_options)
+      end
+
+      if user != ssh_user
+        ssh_result("sudo -n chown -R #{user} #{remote_file}", host, ssh_user)
+      end
+
+      return true
+    rescue Errno::ENOENT => ee
+      raise MessageError.new("Net::SCP was unable to find a private key to use for SSH authenticaton. Try creating a private keyfile or setting up ssh-agent.")
+    rescue OpenSSL::PKey::RSAError
+      raise MessageError.new(connection_error)
+    rescue Net::SSH::AuthenticationFailed
+      raise MessageError.new(connection_error)
+    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+      raise MessageError.new(connection_error)
+    rescue Timeout::Error
+      raise MessageError.new(connection_error)
+    rescue Exception => e
+      raise RemoteCommandError.new(user, host, "scp #{local_file} #{ssh_user}@#{host}:#{remote_file}", nil, '')
+    end
+  end
+  
+  def scp_download(remote_file, local_file, host, user)
+    if is_localhost?(host) && user == whoami()
+      debug("Copy #{remote_file} to #{local_file}")
+      return FileUtils.cp(remote_file, local_file)
+    end
+
+    begin
+      exists = ssh_result("if [ -f #{remote_file} ]; then echo 0; else echo 1; fi", host, user)
+      if exists == "1"
+        raise MessageError.new("Unable to download '#{remote_file}' because the file does not exist on #{host}")
+      end
+    rescue CommandError
+      raise MessageError.new("Unable to check if '#{remote_file}' exists on #{host}")
+    end
+
+    self.synchronize() {
+      unless defined?(Net::SCP)
+        begin
+          require "openssl"
+        rescue LoadError
+          raise("Unable to find the Ruby openssl library. Try installing the openssl package for your version of Ruby (libopenssl-ruby#{RUBY_VERSION[0,3]}).")
+        end
+        require 'net/scp'
+      end
+    }
+
+    ssh_user = get_ssh_user(user)
+    if user != ssh_user
+      debug("SCP user changed to #{ssh_user}")
+    end
+
+    connection_error = "Net::SCP was unable to copy to #{host}:#{remote_file} #{local_file} as #{ssh_user}.  Check that #{host} is online, #{ssh_user} exists and your SSH private keyfile or ssh-agent settings. Try adding --net-ssh-option=port=<SSH port number> if you are using an SSH port other than 22.  Review http://docs.continuent.com/helpwithsshandtpm for more help on diagnosing SSH problems."
+    debug("Copy #{host}:#{remote_file} to #{local_file} as #{ssh_user}")
+    begin
+      Net::SCP.download!(host, ssh_user, remote_file, local_file, get_ssh_options)
+
+      return true
+    rescue Errno::ENOENT => ee
+      raise MessageError.new("Net::SCP was unable to find a private key to use for SSH authenticaton. Try creating a private keyfile or setting up ssh-agent.")
+    rescue OpenSSL::PKey::RSAError
+      raise MessageError.new(connection_error)
+    rescue Net::SSH::AuthenticationFailed
+      raise MessageError.new(connection_error)
+    rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+      raise MessageError.new(connection_error)
+    rescue Timeout::Error
+      raise MessageError.new(connection_error)
+    rescue Exception => e
+      raise RemoteCommandError.new(user, host, "scp #{ssh_user}@#{host}:#{remote_file} #{local_file}", nil, '')
+    end
+  end
 end
 
 module TungstenScript

data/lib/ipparse.rb

@@ -0,0 +1,144 @@
+# VMware Continuent Tungsten Replicator
+# Copyright (C) 2015 VMware, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Initial developer(s): Jeff Mace
+
+class IPParse
+  IPV4 = :ipv4
+  IPV6 = :ipv6
+  
+  def initialize
+    @klass = get_platform_klass().new()
+    @parsed = nil
+  end
+  
+  # Return a nested array of the parsed interface information
+  # { 
+  #   "eth0" => {
+  #     :ipv4 => { :address => "1.2.3.4", :netmask => "255.255.255.0" },
+  #     :ipv6 => { :address => "fe80::a00:27ff:fe1f:84a5", :netmask => "64" }
+  #   }
+  # }
+  def get_interfaces
+    parse()
+    return @parsed
+  end
+  
+  def get_interface_address(interface, type = IPV4)
+    parse()
+    
+    unless @parsed.has_key?(interface)
+      return nil
+    end
+    
+    unless @parsed[interface].has_key?(type)
+      return nil
+    end
+    
+    return @parsed[interface][type]
+  end
+  
+  private
+  
+  def get_platform
+    RUBY_PLATFORM
+  end
+  
+  # Have the platform class parse the IP configuration information 
+  # it just collected. If the information has already been parsed
+  # the steps can be skipped.
+  def parse
+    if @parsed != nil
+      return
+    end
+    
+    @parsed = @klass.parse(get_raw_ip_configuration())
+  end
+  
+  # Have the platform class to collect IP configuration information
+  def get_raw_ip_configuration
+    @klass.get_raw_ip_configuration()
+  end
+  
+  # Iterate through all of the platform classes until one declares 
+  # it is able to support the current system.
+  def get_platform_klass(platform = nil)
+    if platform == nil
+      platform = get_platform()
+    end
+    
+    platform = platform.downcase()
+    
+    IPParsePlatform.subclasses().each{
+      |klass|
+      begin
+        supports_platform = klass.supports_platform?(platform)
+        if supports_platform == true
+          return klass
+        end
+      rescue NoMethodError
+        # Eat this error because the platform didn't define the required method
+      end
+    }
+    
+    throw Exception.new("Unable to support the #{platform} platform")
+  end
+end
+
+class IPParsePlatform
+  def initialize
+    @interfaces = {}
+  end
+  
+  def add_ipv4(name, address, netmask)
+    unless @interfaces.has_key?(name)
+      @interfaces[name] = {}
+    end
+    
+    @interfaces[name][IPParse::IPV4] = {
+      :address => address,
+      :netmask => netmask
+    }
+  end
+  
+  def add_ipv6(name, address, netmask)
+    unless @interfaces.has_key?(name)
+      @interfaces[name] = {}
+    end
+    
+    @interfaces[name][IPParse::IPV6] = {
+      :address => address,
+      :netmask => netmask
+    }
+  end
+  
+  def get_raw_ip_configuration
+    throw Exception.new("Undefined method #{self.class.name}:get_raw_ip_configuration")
+  end
+  
+  def self.inherited(subclass)
+    @subclasses ||= []
+    @subclasses << subclass
+  end
+  
+  def self.subclasses
+    @subclasses
+  end
+end
+
+# Load all platform classes so they can be registered
+Dir.glob("#{File.dirname(__FILE__)}/ipparse/platforms/*.rb").each{|platform|
+  require platform
+}

data/lib/ipparse/README

@@ -0,0 +1,18 @@
+About
+===
+
+This library provides functionality to parse the IP information for a host. It 
+is designed with a limited feature set based on what is necessary for the 
+VMware Continuent project.
+
+Platform Specifics
+===
+
+On Linux platforms, the library will only return values where the link encapsulation is 'Ethernet'. This eliminates loopback and specialty devices.
+
+Known Limitations
+===
+
+The ipparse library does not support interfaces that include multiple IPV6 
+addresses on a single interface. It will identify the first address and return 
+that. Any additional addresses will be ignored.

data/lib/ipparse/platforms/linux.rb

@@ -0,0 +1,103 @@
+# VMware Continuent Tungsten Replicator
+# Copyright (C) 2015 VMware, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Initial developer(s): Jeff Mace
+
+class LinuxIPParsePlatform < IPParsePlatform
+  def self.supports_platform?(platform)
+    if platform.downcase() =~ /linux/
+      true
+    else
+      false
+    end
+  end
+  
+  def get_raw_ip_configuration
+    path = `which ifconfig 2>/dev/null`.chomp()
+    if path == ""
+      path = "/sbin/ifconfig"
+    end
+    
+    results = `export LANG=en_US; #{path} -a`
+    if results == false
+      raise "Unable to collect IP configuration from ifconfig"
+    else
+      return results
+    end
+  end
+  
+  def parse(raw)
+    name_regex = Regexp.compile(/^([a-zA-Z0-9]+)/)
+    encapsulation_regex = Regexp.compile(/Link encap:(\S+)/)
+    flags_regex = Regexp.compile(/flags=([0-9]+)\<([A-Z,]+)\>/)
+    inet4_regex1 = Regexp.compile(/inet addr:([0-9\.]+)[ ]+(Bcast:[0-9\.]+ )?[ ]+Mask:([0-9\.]+)/)
+    inet4_regex2 = Regexp.compile(/inet ([0-9\.]+)[ ]+netmask ([0-9\.]+)[ ]+(broadcast [0-9\.]+ )?/)
+    inet6_regex1 = Regexp.compile(/inet6 addr:[ ]*([a-f0-9:]+)\/([0-9]+)/)
+    inet6_regex2 = Regexp.compile(/inet6 ([a-f0-9:]+)[ ]*prefixlen ([0-9]+)/)
+
+    raw.split("\n\n").each{
+      |ifconfig|
+      include_interface = false
+      
+      begin
+        encapsulation = encapsulation_regex.match(ifconfig)[1]
+        
+        if encapsulation.downcase() == "ethernet"
+          include_interface = true
+        end
+      rescue
+        # Catch the exception and move on
+      end
+      
+      begin
+        flags = flags_regex.match(ifconfig)[2]
+        flags = flags.split(",")
+
+        if flags.include?("LOOPBACK") == false
+          include_interface = true
+        end
+      rescue
+        # Catch the exception and move on
+      end
+      
+      if include_interface == false
+        next
+      end
+      
+      name = name_regex.match(ifconfig)[1]
+      if name == nil
+        raise "Unable to parse IP configuration because a valid name does not exist"
+      end
+      
+      m1 = inet4_regex1.match(ifconfig)
+      m2 = inet4_regex2.match(ifconfig)
+      if m1
+        add_ipv4(name, m1[1], m1[3])
+      elsif m2
+        add_ipv4(name, m2[1], m2[2])
+      end
+      
+      m1 = inet6_regex1.match(ifconfig)
+      m2 = inet6_regex2.match(ifconfig)
+      if m1
+        add_ipv6(name, m1[1], m1[2])
+      elsif m2
+        add_ipv6(name, m2[1], m2[2])
+      end
+    }
+    
+    return @interfaces
+  end
+end